diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..e7eecd2 --- /dev/null +++ b/TODO.md @@ -0,0 +1,32 @@ +# TODO + +## QWP UDP Sender + +### Documented limitation: mixing `atNow()` with `atMicros()` / `atNanos()` + +Current behavior in `QwpUdpSender` is to reject this pattern once committed rows already exist for the table: + +1. Write row(s) with `atNow()` (server-assigned designated timestamp). +2. Start a later row and finish it with `atMicros()` or `atNanos()`. + +The sender throws: + +- `schema change in middle of row is not supported` + +Why this happens: + +- `atNow()` does not write the designated timestamp column. +- `atMicros()` / `atNanos()` writes designated timestamp into the empty-name column (`""`). +- With committed rows already present, introducing this column is treated as schema evolution. +- The UDP incremental-estimate policy forbids schema changes in the middle of an in-progress row. + +Current workaround: + +- Use one designated timestamp strategy consistently per table stream: + - always `atNow()`, or + - always `atMicros()` / `atNanos()`. + +Future fix options: + +- Add explicit support for switching designated timestamp strategy mid-stream by pre-materializing designated timestamp schema state, or +- Harmonize designated timestamp handling so `atNow()` and `atMicros()` / `atNanos()` do not diverge schema shape. diff --git a/ci/confs/authenticated/authDb.txt b/ci/confs/authenticated/authDb.txt deleted file mode 100644 index 2e33102..0000000 --- a/ci/confs/authenticated/authDb.txt +++ /dev/null @@ -1,18 +0,0 @@ -# Test auth db file, format is -# [key/user id] [key type] {key details} ... -# Only elliptic curve (for curve P-256) are supported (key type ec-p-256-sha256), the key details for such a key are the base64url encoded x and y points that determine the public key as defined in the JSON web token standard (RFC 7519) -# -# The auth db file needs to be put somewhere in the questdbn server root and referenced in the line.tcp.auth.db.path setting of server conf, like: -# line.tcp.auth.db.path=conf/authDb.txt -# -# Below is an elliptic curve (for curve P-256) JSON Web Key -#{ -# "kty": "EC", -# "d": "5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48", -# "crv": "P-256", -# "kid": "testUser1", -# "x": "fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU", -# "y": "Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac" -#} -# For this kind of key the "d" parameter is used to generate the secret key. The "x" and "y" parameters are used to generate the public key -testUser1 ec-p-256-sha256 AKfkxOBlqBN8uDfTxu2Oo6iNsOPBnXkEH4gt44tBJKCY AL7WVjoH-IfeX_CXo5G1xXKp_PqHUrdo3xeRyDuWNbBX diff --git a/ci/confs/authenticated/server.conf b/ci/confs/authenticated/server.conf deleted file mode 100644 index d30f878..0000000 --- a/ci/confs/authenticated/server.conf +++ /dev/null @@ -1,1396 +0,0 @@ -# Comment or set to false to allow QuestDB to start even in the presence of config errors. -config.validation.strict=true - -# toggle whether worker should stop on error -#shared.worker.haltOnError=false - -# Number of threads in Network shared thread pool. Network thread pool used for handling HTTP, TCP, UDP and Postgres connections unless dedicated thread pools are configured. -#shared.network.worker.count=2 - -# Comma-delimited list of CPU ids, one per thread specified in "shared.network.worker.count" used for Network thread pool. By default, threads have no CPU affinity -#shared.network.worker.affinity= - -# Number of threads in Query shared thread pool. Query thread pool used for handling parallel queries, like parallel filters and group by queries. -#shared.query.worker.count=2 - -# Comma-delimited list of CPU ids, one per thread specified in "shared.query.worker.count" used for Query thread pool. By default, threads have no CPU affinity -#shared.query.worker.affinity= - -# Number of threads in Write shared thread pool. Write pool threads are used for running WAL Apply work load to merge data from WAL files into the table -#shared.write.worker.count=2 - -# Comma-delimited list of CPU ids, one per thread specified in "shared.write.worker.count" used for Write thread pool. By default, threads have no CPU affinity -#shared.write.worker.affinity= - -# Default number of worker threads in Network, Query and Write shared pools. Single value to configure all three pools sizes. -#shared.worker.count=2 - -# RAM usage limit, as a percentage of total system RAM. A zero value does not -# set any limit. The default is 90. -#ram.usage.limit.percent=90 - -# RAM usage limit, in bytes. A zero value (the default) does not set any limit. -# If both this and ram.usage.limit.percent are non-zero, the lower limit takes precedence. -#ram.usage.limit.bytes=0 - -# Repeats compatible migrations from the specified version. The default setting of 426 allows to upgrade and downgrade QuestDB in the range of versions from 6.2.0 to 7.0.2. -# If set to -1 start time improves but downgrades to versions below 7.0.2 and subsequent upgrades can lead to data corruption and crashes. -#cairo.repeat.migration.from.version=426 - -################ HTTP settings ################## - -# enable HTTP server -http.enabled=false - -# IP address and port of HTTP server -#http.net.bind.to=0.0.0.0:9000 - -# Uncomment to enable HTTP Basic authentication -#http.user=admin -#http.password=quest - -# Maximum time interval for the HTTP server to accept connections greedily in a loop. -# The accept loop timeout is set in milliseconds. -#http.net.accept.loop.timeout=500 - -#http.net.connection.limit=256 -# Windows OS might have a limit on TCP backlog size. Typically Windows 10 has max of 200. This -# means that even if net.connection.limit is set over 200 it wont be possible to have this many -# concurrent connections. To overcome this limitation Windows has an unreliable hack, which you can -# exercise with this flag. Do only set it if you actively try to overcome limit you have already -# experienced. Read more about SOMAXCONN_HINT here https://docs.microsoft.com/en-us/windows/win32/api/winsock2/nf-winsock2-listen -#http.net.connection.hint=false - - -# Maximum HTTP connections that can be used for ILP ingestion using /write http endpoint. This limit must be lower or equal to http.net.connection.limit -# Not restricted by default. Database restart is NOT required when this setting is changed -#http.ilp.connection.limit=-1 - -# Maximum HTTP connections that can be used for queries using /query http endpoint. This limit must be lower or equal to http.net.connection.limit -# Not restricted by default. Database restart is NOT required when this setting is changed -#http.json.query.connection.limit=-1 - -# Maximum HTTP connections that can be used for export using /exp http endpoint. This limit must be lower or equal to http.net.connection.limit -# Restricted to number or CPUs or 25% of overall http connections, whichever is lower, by default. Database restart is NOT required when this setting is changed -#http.export.connection.limit=-1 - -# Idle HTTP connection timeout in milliseconds. -#http.net.connection.timeout=5m - -#Amount of time in milliseconds a connection can wait in the listen backlog queue before its refused. Connections will be aggressively removed from the backlog until the active connection limit is breached -#http.net.connection.queue.timeout=5000 - -# SO_SNDBUF value, -1 = OS default -#http.net.connection.sndbuf=2m - -# SO_RCVBUF value, -1 = OS default -#http.net.connection.rcvbuf=2m - -# size of receive buffer on application side -#http.receive.buffer.size=1m - -# initial size of the connection pool -#http.connection.pool.initial.capacity=4 - -# initial size of the string pool shared by HttpHeaderParser and HttpMultipartContentParser -#http.connection.string.pool.capacity=128 - -# HeaderParser buffer size in bytes -#http.multipart.header.buffer.size=512 - -# how long code accumulates incoming data chunks for column and delimiter analysis -#http.multipart.idle.spin.count=10000 - -#http.request.header.buffer.size=64k - -#http.worker.count=0 -#http.worker.affinity= -#http.worker.haltOnError=false - -# size of send data buffer -#http.send.buffer.size=2m - -# sets the clock to always return zero -#http.frozen.clock=false - -#http.allow.deflate.before.send=false - -# HTTP session timeout -#http.session.timeout=30m - -## When you using SSH tunnel you might want to configure -## QuestDB HTTP server to switch to HTTP/1.0 - -## Set HTTP protocol version to HTTP/1.0 -#http.version=HTTP/1.1 -## Set server keep alive to 'false'. This will make server disconnect client after -## completion of each request -#http.server.keep.alive=true - -## When in HTTP/1.0 mode keep alive values must be 0 -#http.keep-alive.timeout=5 -#http.keep-alive.max=10000 - -#http.static.public.directory=public - -#http.text.date.adapter.pool.capacity=16 -#http.text.json.cache.limit=16384 -#http.text.json.cache.size=8192 -#http.text.max.required.delimiter.stddev=0.1222d -#http.text.max.required.line.length.stddev=0.8 -#http.text.metadata.string.pool.capacity=128 -#http.text.roll.buffer.limit=8216576 -#http.text.roll.buffer.size=1024 -#http.text.analysis.max.lines=1000 -#http.text.lexer.string.pool.capacity=64 -#http.text.timestamp.adapter.pool.capacity=64 -#http.text.utf8.sink.size=4096 - -#http.json.query.connection.check.frequency=1000000 - -# enables the query cache -#http.query.cache.enabled=true - -# sets the number of blocks for the query cache. Cache capacity is number_of_blocks * number_of_rows. -#http.query.cache.block.count= 8 * worker_count - -# sets the number of rows for the query cache. Cache capacity is number_of_blocks * number_of_rows. -#http.query.cache.row.count= 2 * worker_count - -# sets the /settings endpoint readonly -#http.settings.readonly=false - -#http.security.readonly=false -#http.security.max.response.rows=Long.MAX_VALUE - -# Context path for the Web Console. If other REST services remain on the -# default context paths they will move to the same context path as the Web Console. -# Exception is ILP HTTP services, which are not used by the Web Console. They will -# remain on their default context paths. When the default context paths are changed, -# moving the Web Console will not affect the configured paths. QuestDB will create -# a copy of those services on the paths used by the Web Console so the outcome is -# both the Web Console and the custom service are operational. -#http.context.web.console=/ - -# Context path of the file import service -#http.context.import=/imp -# This service is used by the import UI in the Web Console -#http.context.table.status=/chk - -# Context path of the SQL result CSV export service -#http.context.export=/exp - -# This service provides server-side settings to the Web Console -#http.context.settings=/settings - -# SQL execution service -#http.context.execute=/exec - -# Web Console specific service -#http.context.warnings=/warnings - -# ILP HTTP Services. These are not used by the Web Console -#http.context.ilp=/write,/api/v2/write -#http.context.ilp.ping=/ping - -# Custom HTTP redirect service. All redirects are 301 - Moved permanently -#http.redirect.count=1 -#http.redirect.1=/ -> /index.html - -# circuit breaker is a mechanism that interrupts query execution -# at present queries are interrupted when remote client disconnects or when execution takes too long -# and times out - -# circuit breaker is designed to be invoke continuously in a tight loop -# the throttle is a number of pin cycles before abort conditions are tested -#circuit.breaker.throttle=2000000 - -# buffer used by I/O dispatchers and circuit breakers to check the socket state, please do not change this value -# the check reads \r\n from the input stream and discards it since some HTTP clients send this as a keep alive in between requests -#net.test.connection.buffer.size=64 - -# max execution time for read-only query in seconds -# "insert" type of queries are not aborted unless they -# it is "insert as select", where select takes long time before producing rows for the insert -query.timeout=5s - -## HTTP MIN settings -## -## Use this port to health check QuestDB instance when it isn't desired to log these health check requests. This is sort of /dev/null for monitoring - -#http.min.enabled=true -#http.min.net.bind.to=0.0.0.0:9003 - -# When enabled, health check will return HTTP 500 if there were any unhandled errors since QuestDB instance start. -#http.pessimistic.health.check.enabled=false - -# Maximum time interval for the HTTP MIN server to accept connections greedily in a loop. -# The accept loop timeout is set in milliseconds. -#http.min.net.accept.loop.timeout=500 - -################ Cairo settings ################## - -# directory for storing db tables and metadata. this directory is inside the server root directory provided at startup -#cairo.root=db - -# how changes to table are flushed to disk upon commit - default: nosync. Choices: nosync, async (flush call schedules update, returns immediately), sync (waits for flush to complete) -#cairo.commit.mode=nosync - -# The amount of time server is allowed to be compounding transaction before physical commit is forced. -# Compounding of the transactions improves system's through but hurts latency. Reduce this value to -# reduce latency of data visibility. -#cairo.commit.latency=30s - -# number of types table creation or insertion will be attempted -#cairo.create.as.select.retry.count=5 - -# comma separated list of volume definitions, volume_alias -> absolute_path_to_existing_directory. -# volume alias can then be used in create table statement with IN VOLUME clause -#cairo.volumes= by default IN VOLUME is switched off, no volume definitions. - -# type of map uses. Options: 1. fast (speed at the expense of storage. this is the default option) 2. compact -#cairo.default.map.type=fast - -# when true, symbol values will be cached on Java heap -#cairo.default.symbol.cache.flag=false - -# when column type is SYMBOL this parameter specifies approximate capacity for symbol map. -# It should be equal to number of unique symbol values stored in the table and getting this -# value badly wrong will cause performance degradation. Must be power of 2 -#cairo.default.symbol.capacity=256 - -# number of attempts to open files -#cairo.file.operation.retry.count=30 - -# when DB is running in sync/async mode, how many 'steps' IDGenerators will pre-allocate and synchronize sync to disk -#cairo.id.generator.batch.step=512 - -# how often the writer maintenance job gets run -#cairo.idle.check.interval=5m - -# defines the number of latest partitions to keep open when returning a reader to the reader pool -#cairo.inactive.reader.max.open.partitions=128 - -# defines frequency in milliseconds with which the reader pool checks for inactive readers. -#cairo.inactive.reader.ttl=2m - -# defines frequency in milliseconds with which the writer pool checks for inactive readers. -#cairo.inactive.writer.ttl=10m - -# when true (default), TTL enforcement uses wall clock time to prevent accidental data loss -# when future timestamps are inserted. When false, TTL uses only the max timestamp in the table. -#cairo.ttl.use.wall.clock=true - -# approximation of number of rows for single index key, must be power of 2 -#cairo.index.value.block.size=256 - -# number of attempts to open swap file -#cairo.max.swap.file.count=30 - -# file permission for new directories -#cairo.mkdir.mode=509 - -# Time to wait before retrying writing into a table after a memory limit failure -#cairo.write.back.off.timeout.on.mem.pressure=4s - -# maximum file name length in chars. Affects maximum table name length and maximum column name length -#cairo.max.file.name.length=127 - -# minimum number of rows before allowing use of parallel indexation -#cairo.parallel.index.threshold=100000 - -# number of segments in the TableReader pool; each segment holds up to 32 readers -#cairo.reader.pool.max.segments=10 - -# timeout in milliseconds when attempting to get atomic memory snapshots, e.g. in BitmapIndexReaders -#cairo.spin.lock.timeout=1s - -# sets size of the CharacterStore -#cairo.character.store.capacity=1024 - -# Sets size of the CharacterSequence pool -#cairo.character.store.sequence.pool.capacity=64 - -# sets size of the Column pool in the SqlCompiler -#cairo.column.pool.capacity=4096 - -# size of the ExpressionNode pool in SqlCompiler -#cairo.expression.pool.capacity=8192 - -# load factor for all FastMaps -#cairo.fast.map.load.factor=0.7 - -# size of the JoinContext pool in SqlCompiler -#cairo.sql.join.context.pool.capacity=64 - -# size of FloatingSequence pool in GenericLexer -#cairo.lexer.pool.capacity=2048 - -# sets the key capacity in FastMap and CompactMap -#cairo.sql.map.key.capacity=2097152 - -# sets the key capacity in FastMap and CompactMap used in certain queries, e.g. SAMPLE BY -#cairo.sql.small.map.key.capacity=32 - -# number of map resizes in FastMap and CompactMap before a resource limit exception is thrown, each resize doubles the previous size -#cairo.sql.map.max.resizes=2^31 - -# memory page size for FastMap and CompactMap -#cairo.sql.map.page.size=4m - -# memory page size in FastMap and CompactMap used in certain queries, e.g. SAMPLE BY -#cairo.sql.small.map.page.size=32k - -# memory max pages for CompactMap -#cairo.sql.map.max.pages=2^31 - -# sets the size of the QueryModel pool in the SqlCompiler -#cairo.model.pool.capacity=1024 - -# sets the maximum allowed negative value used in LIMIT clause in queries with filters -#cairo.sql.max.negative.limit=10000 - -# sets the memory page size for storing keys in LongTreeChain -#cairo.sql.sort.key.page.size=128k - -# max number of pages for storing keys in LongTreeChain before a resource limit exception is thrown -# cairo.sql.sort.key.max.pages=2^31 - -# sets the memory page size and max pages for storing values in LongTreeChain -#cairo.sql.sort.light.value.page.size=128k -#cairo.sql.sort.light.value.max.pages=2^31 - -# sets the memory page size and max pages of the slave chain in full hash joins -#cairo.sql.hash.join.value.page.size=16777216 -#cairo.sql.hash.join.value.max.pages=2^31 - -# sets the initial capacity for row id list used for latest by -#cairo.sql.latest.by.row.count=1000 - -# sets the memory page size and max pages of the slave chain in light hash joins -#cairo.sql.hash.join.light.value.page.size=128k -#cairo.sql.hash.join.light.value.max.pages=2^31 - -# number of rows to scan linearly before starting binary search in ASOF JOIN queries with no additional keys -#cairo.sql.asof.join.lookahead=10 - -# sets memory page size and max pages of file storing values in SortedRecordCursorFactory -#cairo.sql.sort.value.page.size=16777216 -#cairo.sql.sort.value.max.pages=2^31 - -# latch await timeout in nanoseconds for stealing indexing work from other threads -#cairo.work.steal.timeout.nanos=10000 - -# whether parallel indexation is allowed. Works in conjunction with cairo.parallel.index.threshold -#cairo.parallel.indexing.enabled=true - -# memory page size for JoinMetadata file -#cairo.sql.join.metadata.page.size=16384 - -# number of map resizes in JoinMetadata before a resource limit exception is thrown, each resize doubles the previous size -#cairo.sql.join.metadata.max.resizes=2^31 - -# size of PivotColumn pool in SqlParser -#cairo.sql.pivot.column.pool.capacity=64 - -# maximum number of columns PIVOT can produce (FOR value combinations × aggregates) -#cairo.sql.pivot.max.produced.columns=5000 - -# size of WindowColumn pool in SqlParser -#cairo.sql.window.column.pool.capacity=64 - -# sets the memory page size and max number of pages for records in window function -#cairo.sql.window.store.page.size=1m -#cairo.sql.window.store.max.pages=2^31 - -# sets the memory page size and max number of pages for row ids in window function -#cairo.sql.window.rowid.page.size=512k -#cairo.sql.window.rowid.max.pages=2^31 - -# sets the memory page size and max number of pages for keys in window function -#cairo.sql.window.tree.page.size=512k -#cairo.sql.window.tree.max.pages=2^31 - -# sets initial size of per-partition window function range frame buffer -#cairo.sql.window.initial.range.buffer.size=32 - -# batch size of non-atomic inserts for CREATE TABLE AS SELECT statements -#cairo.sql.create.table.model.batch.size=1000000 - -# Size of the pool for model objects, that underpin the "create table" parser. -# It is aimed at reducing allocations and it a performance setting. The -# number is aligned to the max concurrent "create table" requests the system -# will ever receive. If system receives more requests that this, it will just -# allocate more object and free them after use. -#cairo.create.table.column.model.pool.capacity=16 - -# size of RenameTableModel pool in SqlParser -#cairo.sql.rename.table.model.pool.capacity=16 - -# size of WithClauseModel pool in SqlParser -#cairo.sql.with.clause.model.pool.capacity=128 - -# size of CompileModel pool in SqlParser -#cairo.sql.compile.view.model.pool.capacity=8 - -# initial size of view lexer pool in SqlParser, used to parse SELECT statements of view definitions -# the max number of views used in a single query determines how many view lexers should be in the pool -#cairo.sql.view.lexer.pool.capacity=8 - -# size of InsertModel pool in SqlParser -#cairo.sql.insert.model.pool.capacity=64 - -# batch size of non-atomic inserts for INSERT INTO SELECT statements -#cairo.sql.insert.model.batch.size=1000000 - -# enables parallel GROUP BY execution; by default, parallel GROUP BY requires at least 4 shared worker threads to take place -#cairo.sql.parallel.groupby.enabled=true - -# merge queue capacity for parallel GROUP BY; used for parallel tasks that merge shard hash tables -#cairo.sql.parallel.groupby.merge.shard.queue.capacity= - -# threshold for parallel GROUP BY to shard the hash table holding the aggregates -#cairo.sql.parallel.groupby.sharding.threshold=10000 - -# enables statistics-based hash table pre-sizing in parallel GROUP BY -#cairo.sql.parallel.groupby.presize.enabled=true - -# maximum allowed hash table size for parallel GROUP BY hash table pre-sizing -#cairo.sql.parallel.groupby.presize.max.capacity=100000000 - -# maximum allowed heap size for parallel GROUP BY hash table pre-sizing -#cairo.sql.parallel.groupby.presize.max.heap.size=1G - -# threshold for parallel ORDER BY + LIMIT execution on sharded GROUP BY hash table -#cairo.sql.parallel.groupby.topk.threshold=5000000 - -# queue capacity for parallel ORDER BY + LIMIT applied to sharded GROUP BY -#cairo.sql.parallel.groupby.topk.queue.capacity= - -# threshold for in-flight tasks for disabling work stealing during parallel SQL execution -# when the number of shared workers is less than 4x of this setting, work stealing is always enabled -#cairo.sql.parallel.work.stealing.threshold=16 - -# spin timeout in nanoseconds for adaptive work stealing strategy -# controls how long the query thread waits for worker threads to pick up tasks before stealing work back -#cairo.sql.parallel.work.stealing.spin.timeout=50000 - -# enables parallel read_parquet() SQL function execution; by default, parallel read_parquet() requires at least 4 shared worker threads to take place -#cairo.sql.parallel.read.parquet.enabled=true - -# capacity for Parquet page frame cache; larger values may lead to better ORDER BY and some other -# clauses performance at the cost of memory overhead -#cairo.sql.parquet.frame.cache.capacity=3 - -# default size for memory buffers in GROUP BY function native memory allocator -#cairo.sql.groupby.allocator.default.chunk.size=128K - -# maximum allowed native memory allocation for GROUP BY functions -#cairo.sql.groupby.allocator.max.chunk.size=4G - -# threshold in bytes for switching from single memory buffer hash table (unordered) to a hash table with separate heap for entries (ordered) -#cairo.sql.unordered.map.max.entry.size=32 - -## prevents stack overflow errors when evaluating complex nested SQLs -## the value is an approximate number of nested SELECT clauses. -#cairo.sql.window.max.recursion=128 - -## pre-sizes the internal data structure that stores active query executions -## the value is chosen automatically based on the number of threads in the shared worker pool -#cairo.sql.query.registry.pool.size= - -## window function buffer size in record counts -## pre-sizes buffer for every windows function execution to contain window records -#cairo.sql.analytic.initial.range.buffer.size=32 - -## enables quick and radix sort in order by, when applicable -#cairo.sql.orderby.sort.enabled=true - -## defines number of rows to use radix sort in order by -#cairo.sql.orderby.radix.sort.threshold=600 - -## enables the column alias to be generated from the expression -#cairo.sql.column.alias.expression.enabled=true - -## maximum length of generated column aliases -#cairo.sql.column.alias.generated.max.size=64 - -## initial capacity of string pool for preferences store and parser -#cairo.preferences.string.pool.capacity=64 - -## Flag to enable or disable symbol capacity auto-scaling. Auto-scaling means resizing -## symbol table data structures as the number of symbols in the table grows. Optimal sizing of -## these data structures ensures optimal ingres performance. -## -## By default, the auto-scaling is enabled. This is optimal. You may want to disable auto-scaling in case -## something goes wrong. -## -## Database restart is NOT required when this setting is changed, but `reload_config()` SQL should be executed. -#cairo.auto.scale.symbol.capacity=true - -## Symbol occupancy threshold after which symbol capacity is doubled. For example -## threshold of 0.8 means that occupancy have to reach 80% of capacity before capacity is increased -#cairo.auto.scale.symbol.capacity.threshold=0.8 - -#### SQL COPY - -# size of CopyModel pool in SqlParser -#cairo.sql.copy.model.pool.capacity=32 - -# size of buffer used when copying tables -#cairo.sql.copy.buffer.size=2m - -# name of file with user's set of date and timestamp formats -#cairo.sql.copy.formats.file=/text_loader.json - -# input root directory, where COPY command and read_parquet() function read files from -# relative paths are resolved against the server root directory -cairo.sql.copy.root=import - -# export root directory, where COPY .. to command write files to -# relative paths are resolved against the server root directory -#cairo.sql.copy.export.root=export - -# input work directory, where temporary import files are created, by default it's located in tmp directory inside the server root directory -#cairo.sql.copy.work.root=null - -# default max size of intermediate import file index chunk (100MB). Import shouldn't use more memory than worker_count * this . -#cairo.sql.copy.max.index.chunk.size=100M - -# Capacity of the internal queue used to split parallel copy SQL command into subtasks and execute them across shared worker threads. -# The default configuration should be suitable for importing files of any size. -#cairo.sql.copy.queue.capacity=32 - -# Capacity of the internal queue used to execute copy export SQL command. -#cairo.sql.copy.export.queue.capacity=32 - -# Frequency of logging progress when exporting data using COPY .. TO command to export to parquet format. 0 or negative value disables the logging. -# Database restart is NOT required when this setting is changed -#cairo.parquet.export.copy.report.frequency.lines=50000 - -# Parquet version to use when exporting data using COPY .. TO command. Valid values: 1 (PARQUET_1_0), 2 (PARQUET_2_LATEST) -#cairo.parquet.export.version=1 - -# Enable statistics collection in Parquet files when exporting data using COPY .. TO command -#cairo.parquet.export.statistics.enabled=true - -# Enable raw array encoding for repeated fields in Parquet files when exporting data using COPY .. TO command -#cairo.parquet.export.raw.array.encoding.enabled=false - -# Compression codec to use when exporting data using COPY .. TO command. Valid values: -# UNCOMPRESSED, SNAPPY, GZIP, BROTLI, ZSTD, LZ4_RAW -#cairo.parquet.export.compression.codec=ZSTD - -# Compression level for GZIP (0-9), BROTLI (0-11), or ZSTD (1-22) codecs when exporting data using COPY .. TO command -#cairo.parquet.export.compression.level=9 - -# Row group size in rows when exporting data using COPY .. TO command. 0 uses the default (512*512 rows) -# Database restart is NOT required when this setting is changed -#cairo.parquet.export.row.group.size=0 - -# Data page size in bytes when exporting data using COPY .. TO command. 0 uses the default (1024*1024 bytes) -# Database restart is NOT required when this setting is changed -#cairo.parquet.export.data.page.size=0 - -# Maximum time to wait for export to complete when using /exp HTTP endpoint. 0 means no timeout. -#http.export.timeout=300s - -# Parquet version to use for partition encoder (for storing partitions in parquet format). Valid values: 1 (PARQUET_1_0), 2 (PARQUET_2_LATEST) -#cairo.partition.encoder.parquet.version=1 - -# Enable statistics collection in Parquet files for partition encoder -#cairo.partition.encoder.parquet.statistics.enabled=true - -# Enable raw array encoding for repeated fields in Parquet files for partition encoder -#cairo.partition.encoder.parquet.raw.array.encoding.enabled=false - -# Compression codec to use for partition encoder. Valid values: -# UNCOMPRESSED, SNAPPY, GZIP, BROTLI, ZSTD, LZ4_RAW -#cairo.partition.encoder.parquet.compression.codec=ZSTD - -# Compression level for GZIP (0-9), BROTLI (0-11), or ZSTD (1-22) codecs for partition encoder -#cairo.partition.encoder.parquet.compression.level=9 - -# Row group size in rows for partition encoder -#cairo.partition.encoder.parquet.row.group.size=100000 - -# Data page size in bytes for partition encoder -#cairo.partition.encoder.parquet.data.page.size=1048576 - -# Number of days to retain records in import log table (sys.parallel_text_import_log). Old records get deleted on each import and server restart. -#cairo.sql.copy.log.retention.days=3 - -# output root directory for backups -#cairo.sql.backup.root=null - -# date format for backup directory -#cairo.sql.backup.dir.datetime.format=yyyy-MM-dd - -# name of temp directory used during backup -#cairo.sql.backup.dir.tmp.name=tmp - -# permission used when creating backup directories -#cairo.sql.backup.mkdir.mode=509 - -# suffix of the partition directory in detached root to indicate it is ready to be attached -#cairo.attach.partition.suffix=.attachable - -# Use file system "copy" operation instead of "hard link" when attaching partition from detached root. Set to true if detached root is on a different drive. -#cairo.attach.partition.copy=false - -# file permission used when creating detached directories -#cairo.detached.mkdir.mode=509 - -# sample by index query page size - max values returned in single scan -# 0 means to use symbol block capacity -# cairo.sql.sampleby.page.size=0 - -# sample by default alignment -# if true, sample by will default to SAMPLE BY (FILL) ALIGN TO CALENDAR -# if false, sample by will default to SAMPLE BY (FILL) ALIGN TO FIRST OBSERVATION -# cairo.sql.sampleby.default.alignment.calendar=true - -# sets the minimum number of rows in page frames used in SQL queries -#cairo.sql.page.frame.min.rows=100000 - -# sets the maximum number of rows in page frames used in SQL queries -#cairo.sql.page.frame.max.rows=1000000 - -# sets the minimum number of rows in small page frames used in SQL queries, primarily for window joins -#cairo.sql.small.page.frame.min.rows=10000 - -# sets the maximum number of rows in small page frames used in SQL queries, primarily for window joins -#cairo.sql.small.page.frame.max.rows=100000 - -# sets the memory page size and max number of pages for memory used by rnd functions -# currently rnd_str() and rnd_symbol(), this could extend to other rnd functions in the future -#cairo.rnd.memory.page.size=8K -#cairo.rnd.memory.max.pages=128 - -# max length (in chars) of buffer used to store result of SQL functions, such as replace() or lpad() -#cairo.sql.string.function.buffer.max.size=1048576 - -# SQL JIT compiler mode. Options: -# 1. on (enable JIT and use vector instructions when possible; default value) -# 2. scalar (enable JIT and use scalar instructions only) -# 3. off (disable JIT) -#cairo.sql.jit.mode=on - -# sets the memory page size and max pages for storing IR for JIT compilation -#cairo.sql.jit.ir.memory.page.size=8K -#cairo.sql.jit.ir.memory.max.pages=8 - -# sets the memory page size and max pages for storing bind variable values for JIT compiled filter -#cairo.sql.jit.bind.vars.memory.page.size=4K -#cairo.sql.jit.bind.vars.memory.max.pages=8 - -# sets debug flag for JIT compilation; when enabled, assembly will be printed into stdout -#cairo.sql.jit.debug.enabled=false - -# Controls the maximum allowed IN list length before the JIT compiler will fall back to Java code -#cairo.sql.jit.max.in.list.size.threshold=10 - -#cairo.date.locale=en - -# Maximum number of uncommitted rows in TCP ILP -#cairo.max.uncommitted.rows=500000 - -# Minimum size of in-memory buffer in milliseconds. The buffer is allocated dynamically through analysing -# the shape of the incoming data, and o3MinLag is the lower limit -#cairo.o3.min.lag=1s - -# Maximum size of in-memory buffer in milliseconds. The buffer is allocated dynamically through analysing -# the shape of the incoming data, and o3MaxLag is the upper limit -#cairo.o3.max.lag=600000 - -# Memory page size per column for O3 operations. Please be aware O3 will use 2x of this RAM per column -#cairo.o3.column.memory.size=8M - -# Memory page size per column for O3 operations on System tables only -#cairo.system.o3.column.memory.size=256k - -# Number of partition expected on average, initial value for purge allocation job, extended in runtime automatically -#cairo.o3.partition.purge.list.initial.capacity=1 - -# mmap sliding page size that TableWriter uses to append data for each column -#cairo.writer.data.append.page.size=16M - -# mmap page size for mapping small files, such as _txn, _todo and _meta -# the default value is OS page size (4k Linux, 64K windows, 16k OSX M1) -# if you override this value it will be rounded to the nearest (greater) multiple of OS page size -#cairo.writer.misc.append.page.size=4k - -# mmap page size for appending index key data; key data is number of distinct symbol values times 4 bytes -#cairo.writer.data.index.key.append.page.size=512k - -# mmap page size for appending value data; value data are rowids, e.g. number of rows in partition times 8 bytes -#cairo.writer.data.index.value.append.page.size=16M - -# mmap sliding page size that TableWriter uses to append data for each column specifically for System tables -#cairo.system.writer.data.append.page.size=256k - -# File allocation page min size for symbol table files -#cairo.symbol.table.min.allocation.page.size=4k - -# File allocation page max size for symbol table files -#cairo.symbol.table.max.allocation.page.size=8M - -# Maximum wait timeout in milliseconds for ALTER TABLE SQL statement run via REST and PG Wire interfaces when statement execution is ASYNCHRONOUS -#cairo.writer.alter.busy.wait.timeout=500ms - -# Row count to check writer command queue after on busy writing (e.g. tick after X rows written) -#cairo.writer.tick.rows.count=1024 - -# Maximum writer ALTER TABLE and replication command capacity. Shared between all the tables -#cairo.writer.command.queue.capacity=32 - -# Sets flag to enable io_uring interface for certain disk I/O operations on newer Linux kernels (5.12+). -#cairo.iouring.enabled=true - -# Minimum O3 partition prefix size for which O3 partition split happens to avoid copying the large prefix -#cairo.o3.partition.split.min.size=50M - -# The number of O3 partition splits allowed for the last partitions. If the number of splits grows above this value, the splits will be squashed -#cairo.o3.last.partition.max.splits=20 - -################ Parallel SQL execution ################ - -# Sets flag to enable parallel SQL filter execution. JIT compilation takes place only when this setting is enabled. -#cairo.sql.parallel.filter.enabled=true - -# Sets the threshold for column pre-touch to be run as a part of the parallel SQL filter execution. The threshold defines ratio between the numbers of scanned and filtered rows. -#cairo.sql.parallel.filter.pretouch.threshold=0.05 - -# Sets the upper limit on the number of in-flight tasks for parallel query workers published by filter queries with LIMIT. -#cairo.sql.parallel.filter.dispatch.limit= - -# Sets flag to enable parallel ORDER BY + LIMIT SQL execution. -#cairo.sql.parallel.topk.enabled=true - -# Sets flag to enable parallel WINDOW JOIN SQL execution. -#cairo.sql.parallel.window.join.enabled=true - -# Shard reduce queue contention between SQL statements that are executed concurrently. -#cairo.page.frame.shard.count=4 - -# Reduce queue is used for data processing and should be large enough to supply tasks for worker threads (shared worked pool). -#cairo.page.frame.reduce.queue.capacity= - -# Reduce queue is used for vectorized data processing and should be large enough to supply tasks for worker threads (shared worked pool). -#cairo.vector.aggregate.queue.capacity= - -# Initial row ID list capacity for each slot of the "reduce" queue. Larger values reduce memory allocation rate, but increase RSS size. -#cairo.page.frame.rowid.list.capacity=256 - -# Initial column list capacity for each slot of the "reduce" queue. Used by JIT-compiled filters. -#cairo.page.frame.column.list.capacity=16 - -# Initial object pool capacity for local "reduce" tasks. These tasks are used to avoid blocking query execution when the "reduce" queue is full. -#cairo.page.frame.task.pool.capacity=4 - -################ LINE settings ###################### - -#line.default.partition.by=DAY - -# Enable / Disable automatic creation of new columns in existing tables via ILP. When set to false overrides value of line.auto.create.new.tables to false -#line.auto.create.new.columns=true - -# Enable / Disable automatic creation of new tables via ILP. -#line.auto.create.new.tables=true - -# Enable / Disable printing problematic ILP messages in case of errors. -#line.log.message.on.error=true - -################ LINE UDP settings ################## - -#line.udp.bind.to=0.0.0.0:9009 -#line.udp.join=232.1.2.3 -#line.udp.commit.rate=1000000 -#line.udp.msg.buffer.size=2048 -#line.udp.msg.count=10000 -#line.udp.receive.buffer.size=8m -line.udp.enabled=true -#line.udp.own.thread.affinity=-1 -#line.udp.own.thread=false -#line.udp.unicast=false -#line.udp.commit.mode=nosync -#line.udp.timestamp=n - -######################### LINE TCP settings ############################### - -#line.tcp.enabled=true -#line.tcp.net.bind.to=0.0.0.0:9009 -#line.tcp.net.connection.limit=256 -line.tcp.auth.db.path=conf/authDb.txt - -# Windows OS might have a limit on TCP backlog size. Typically Windows 10 has max of 200. This -# means that even if net.connection.limit is set over 200 it wont be possible to have this many -# concurrent connections. To overcome this limitation Windows has an unreliable hack, which you can -# exercise with this flag. Do only set it if you actively try to overcome limit you have already -# experienced. Read more about SOMAXCONN_HINT here https://docs.microsoft.com/en-us/windows/win32/api/winsock2/nf-winsock2-listen -#line.tcp.net.connection.hint=false - -# Idle TCP connection timeout in milliseconds. 0 means there is no timeout. -#line.tcp.net.connection.timeout=0 - -# Amount of time in milliseconds a connection can wait in the listen backlog queue before its refused. Connections will be aggressively removed from the backlog until the active connection limit is breached -#line.tcp.net.connection.queue.timeout=5000 - -# Maximum time interval for the ILP TCP endpoint to accept connections greedily in a loop. -# The accept loop timeout is set in milliseconds. -#line.tcp.net.accept.loop.timeout=500 - -# SO_RCVBUF value, -1 = OS default -#line.tcp.net.connection.rcvbuf=-1 - -#line.tcp.connection.pool.capacity=64 -#line.tcp.timestamp=n - -# TCP message buffer size -#line.tcp.msg.buffer.size=2048 - -# Max measurement size -#line.tcp.max.measurement.size=2048 - -# Max receive buffer size -#line.tcp.max.recv.buffer.size=1073741824 - -# Size of the queue between the IO jobs and the writer jobs, each queue entry represents a measurement -#line.tcp.writer.queue.capacity=128 - -# IO and writer job worker pool settings, 0 indicates the shared pool should be used -#line.tcp.writer.worker.count=0 -#line.tcp.writer.worker.affinity= -#line.tcp.writer.worker.yield.threshold=10 -#line.tcp.writer.worker.nap.threshold=7000 -#line.tcp.writer.worker.sleep.threshold=10000 -#line.tcp.writer.halt.on.error=false - -#line.tcp.io.worker.count=0 -#line.tcp.io.worker.affinity= -#line.tcp.io.worker.yield.threshold=10 -#line.tcp.io.worker.nap.threshold=7000 -#line.tcp.io.worker.sleep.threshold=10000 -#line.tcp.io.halt.on.error=false - -# Sets flag to disconnect TCP connection that sends malformed messages. -#line.tcp.disconnect.on.error=true - -# Commit lag fraction. Used to calculate commit interval for the table according to the following formula: -# commit_interval = commit_lag ∗ fraction -# The calculated commit interval defines how long uncommitted data will need to remain uncommitted. -#line.tcp.commit.interval.fraction=0.5 -# Default commit interval in milliseconds. Used when o3MinLag is set to 0. -#line.tcp.commit.interval.default=2000 - -# Maximum amount of time in between maintenance jobs in milliseconds, these will commit uncommitted data -#line.tcp.maintenance.job.interval=1000 -# Minimum amount of idle time before a table writer is released in milliseconds -#line.tcp.min.idle.ms.before.writer.release=500 - - -#line.tcp.symbol.cache.wait.before.reload=500ms - -######################### LINE HTTP settings ############################### - -#line.http.enabled=true - -#line.http.max.recv.buffer.size=1073741824 - -#line.http.ping.version=v2.7.4 - -################ PG Wire settings ################## - -#pg.enabled=true -#pg.net.bind.to=0.0.0.0:8812 -#pg.net.connection.limit=64 -# Windows OS might have a limit on TCP backlog size. Typically Windows 10 has max of 200. This -# means that even if active.connection.limit is set over 200 it wont be possible to have this many -# concurrent connections. To overcome this limitation Windows has an unreliable hack, which you can -# exercise with this flag. Do only set it if you actively try to overcome limit you have already -# experienced. Read more about SOMAXCONN_HINT here https://docs.microsoft.com/en-us/windows/win32/api/winsock2/nf-winsock2-listen -#pg.net.connection.hint=false - -# Connection idle timeout in milliseconds. Connections are closed by the server when this timeout lapses. -#pg.net.connection.timeout=300000 - -# Amount of time in milliseconds a connection can wait in the listen backlog queue before its refused. Connections will be aggressively removed from the backlog until the active connection limit is breached. -#pg.net.connection.queue.timeout=5m - -# Maximum time interval for the PG Wire server to accept connections greedily in a loop. -# The accept loop timeout is set in milliseconds. -#pg.net.accept.loop.timeout=500 - -# SO_RCVBUF value, -1 = OS default -#pg.net.connection.rcvbuf=-1 - -# SO_SNDBUF value, -1 = OS default -#pg.net.connection.sndbuf=-1 - -#pg.legacy.mode.enabled=false -#pg.character.store.capacity=4096 -#pg.character.store.pool.capacity=64 -#pg.connection.pool.capacity=64 -#pg.password=quest -#pg.user=admin -# Enables read-only mode for the pg wire protocol. In this mode data mutation queries are rejected. -#pg.security.readonly=false -#pg.readonly.password=quest -#pg.readonly.user=user -# Enables separate read-only user for the pg wire server. Data mutation queries are rejected for all connections opened by this user. -#pg.readonly.user.enabled=false -# enables select query cache -#pg.select.cache.enabled=true -# sets the number of blocks for the select query cache. Cache capacity is number_of_blocks * number_of_rows. -#pg.select.cache.block.count= 8 * worker_count -# sets the number of rows for the select query cache. Cache capacity is number_of_blocks * number_of_rows. -#pg.select.cache.row.count= 2 * worker_count -# enables insert query cache -#pg.insert.cache.enabled=true -# sets the number of blocks for the insert query cache. Cache capacity is number_of_blocks * number_of_rows. -#pg.insert.cache.block.count=4 -# sets the number of rows for the insert query cache. Cache capacity is number_of_blocks * number_of_rows. -#pg.insert.cache.row.count=4 -#pg.max.blob.size.on.query=512k -#pg.recv.buffer.size=1M -#pg.net.connection.rcvbuf=-1 -#pg.send.buffer.size=1M -#pg.net.connection.sndbuf=-1 -#pg.date.locale=en -#pg.worker.count=2 -#pg.worker.affinity=-1,-1; -#pg.halt.on.error=false -#pg.daemon.pool=true -#pg.binary.param.count.capacity=2 -# maximum number of prepared statements a single client can create at a time. This is to prevent clients from creating -# too many prepared statements and exhausting server resources. -#pg.named.statement.limit=10000 - -# if you are using insert batches of over 64 rows, you should increase this value to avoid memory resizes that -# might slow down inserts. Be careful though as this allocates objects on JavaHeap. Setting this value too large -# might kill the GC and server will be unresponsive on startup. -#pg.pipeline.capacity=64 - -################ Materialized View settings ################## - -# Enables SQL support and refresh job for materialized views. -#cairo.mat.view.enabled=true - -# When disabled, SQL executed by materialized view refresh job always runs single-threaded. -#cairo.mat.view.parallel.sql.enabled=true - -# Desired number of base table rows to be scanned by one query during materialized view refresh. -#cairo.mat.view.rows.per.query.estimate=1000000 - -# Maximum time interval used for a single step of materialized view refresh. For larger intervals single SAMPLE BY interval will be used as the step. -#cairo.mat.view.max.refresh.step=31536000000000us - -# Maximum number of times QuestDB will retry a materialized view refresh after a recoverable error. -#cairo.mat.view.max.refresh.retries=10 - -# Timeout for next refresh attempts after an out-of-memory error in a materialized view refresh, in milliseconds. -#cairo.mat.view.refresh.oom.retry.timeout=200 - -# Maximum number of rows inserted by the refresh job in a single transaction. -#cairo.mat.view.insert.as.select.batch.size=1000000 - -# Maximum number of time intervals from WAL data transactions cached per materialized view. -#cairo.mat.view.max.refresh.intervals=100 - -# Interval for periodical refresh intervals caching for materialized views. -#cairo.mat.view.refresh.intervals.update.period=15s - -# Number of parallel threads used to refresh materialized view data. -# Configuration options: -# - Default: Automatically calculated based on CPU core count -# - 0: Runs as a single instance on the shared worker pool -# - N: Uses N dedicated threads for parallel refreshing -#mat.view.refresh.worker.count= - -# Materialized View refresh worker pool configuration -#mat.view.refresh.worker.affinity= -#mat.view.refresh.worker.yield.threshold=1000 -#mat.view.refresh.worker.nap.threshold=7000 -#mat.view.refresh.worker.sleep.threshold=10000 -#mat.view.refresh.worker.haltOnError=false - -# Export worker pool configuration -#export.worker.affinity= -#export.worker.yield.threshold=1000 -#export.worker.nap.threshold=7000 -#export.worker.sleep.threshold=10000 -#export.worker.haltOnError=false - -################ WAL settings ################## - -# Enable support of creating and writing to WAL tables -#cairo.wal.supported=true - -# If set to true WAL becomes the default mode for newly created tables. Impacts table created from ILP and SQL if WAL / BYPASS WAL not specified -#cairo.wal.enabled.default=true - -# Parallel threads to apply WAL data to the table storage. By default it is equal to the CPU core count. -# When set to 0 WAL apply job will run as a single instance on shared worker pool. -#wal.apply.worker.count= - -# WAL apply pool configuration -#wal.apply.worker.affinity= -#wal.apply.worker.yield.threshold=1000 -#wal.apply.worker.nap.threshold=7000 -#wal.apply.worker.sleep.threshold=10000 -#wal.apply.worker.haltOnError=false - -# Period in ms of how often WAL applied files are cleaned up from the disk -#cairo.wal.purge.interval=30s - -# Row count of how many rows are written to the same WAL segment before starting a new segment. -# Triggers in conjunction with `cairo.wal.segment.rollover.size` (whichever is first). -#cairo.wal.segment.rollover.row.count=200000 - -# Byte size of number of rows written to the same WAL segment before starting a new segment. -# Triggers in conjunction with `cairo.wal.segment.rollover.row.count` (whichever is first). -# By default this is 0 (disabled) unless `replication.role=primary` is set, then it is defaulted to 2MiB. -#cairo.wal.segment.rollover.size=0 - -# mmap sliding page size that WalWriter uses to append data for each column -#cairo.wal.writer.data.append.page.size=1M - -# mmap sliding page size that WalWriter uses to append events for each column -# this page size has performance impact on large number of small transactions, larger -# page will cope better. However, if the workload is that of small number of large transaction, -# this page size can be reduced. The optimal value should be established via ingestion benchmark. -#cairo.wal.writer.event.append.page.size=128k - -# Multiplier to cairo.max.uncommitted.rows to calculate the limit of rows that can kept invisible when writing -# to WAL table under heavy load, when multiple transactions are to be applied. -# It is used to reduce the number Out Of Order commits when Out Of Order commits are unavoidable by squashing multiple commits together. -# Setting it very low can increase Out Of Order commit frequency and decrease the throughput. -# No rows are kept in WAL lag when last committed transaction is processed. -#cairo.wal.squash.uncommitted.rows.multiplier=20.0 - -# Maximum size of data can be kept in WAL lag in bytes. -# Default is 75M and it means that once the limit is reached, 50M will be committed and 25M will be kept in the lag. -# It is used to reduce the number Out Of Order commits when Out Of Order commits are unavoidable by squashing multiple commits together. -# Setting it very low can increase Out Of Order commit frequency and decrease the throughput. -# No rows are kept in WAL lag when last committed transaction is processed. -#cairo.wal.max.lag.size=75M - -# Maximum number of transactions to keep in O3 lag for WAL tables. Once the number is reached, full commit occurs. -# By default it is -1 and the limit does not apply, instead the size limit of cairo.wal.max.lag.size is used. -# No rows are kept in WAL lag when last committed transaction is processed. -#cairo.wal.max.lag.txn.count=-1 - -# When WAL apply job processes transactions this is the minimum number of transaction -# to look ahead and read metadata of before applying any of them. -#cairo.wal.apply.look.ahead.txn.count=20 - -# Part of WAL apply job fair factor. The amount of time job spends on single table -# before moving to the next one. -#cairo.wal.apply.table.time.quota=1s - -# number of segments in the WalWriter pool; each segment holds up to 16 writers -#cairo.wal.writer.pool.max.segments=10 - -# number of segments in the ViewWalWriter pool; each segment holds up to 16 writers -#cairo.view.wal.writer.pool.max.segments=4 - -# ViewWalWriter pool releases inactive writers after this TTL. In milliseconds. -#cairo.view.wal.inactive.writer.ttl=60000 - -# Maximum number of Segments to cache File descriptors for when applying from WAL to table storage. -# WAL segment files are cached to avoid opening and closing them on processing each commit. -# Ideally should be in line with average number of simultaneous connections writing to the tables. -#cairo.wal.max.segment.file.descriptors.cache=30 - -# When disabled, SQL executed by WAL apply job always runs single-threaded. -#cairo.wal.apply.parallel.sql.enabled=true - -################ Telemetry settings ################## - -# Telemetry events are used to identify components of QuestDB that are being -# used. They never identify user nor data stored in QuestDB. All events can be -# viewed via `select * from telemetry`. Switching telemetry off will stop all -# events from being collected. After telemetry is switched off, telemetry -# tables can be dropped. - -telemetry.enabled=false -#telemetry.queue.capacity=512 -#telemetry.hide.tables=true -#telemetry.table.ttl.weeks=4 -#telemetry.event.throttle.interval=1m - -################ Metrics settings ################## - -#metrics.enabled=false - -############# Query Tracing settings ############### - -#query.tracing.enabled=false - -################ Logging settings ################## - -# enable or disable 'exe' log messages written when query execution begins -#log.sql.query.progress.exe=true - - -################ Enterprise configuration options ################## -### Please visit https://questdb.io/enterprise/ for more information -#################################################################### - - -#acl.enabled=false -#acl.entity.name.max.length=255 -#acl.admin.user.enabled=true -#acl.admin.user=admin -#acl.admin.password=quest - -## 10 seconds -#acl.rest.token.refresh.threshold=10 -#acl.sql.permission.model.pool.capacity=32 -#acl.password.hash.iteration.count=100000 - -#acl.basic.auth.realm.enabled=false - -#cold.storage.enabled=false -#cold.storage.object.store= - -#tls.enabled=false -#tls.cert.path= -#tls.private.key.path= - -## Defaults to the global TLS settings -## including enable/disable flag and paths -## Use these configuration values to separate -## min-http server TLS configuration from global settings -#http.min.tls.enabled= -#http.min.tls.cert.path= -#http.min.tls.private.key.path= - -#http.tls.enabled=false -#http.tls.cert.path= -#http.tls.private.key.path= - -#line.tcp.tls.enabled= -#line.tcp.tls.cert.path= -#line.tcp.tls.private.key.path= - -#pg.tls.enabled= -#pg.tls.cert.path= -#pg.tls.private.key.path= - -## The number of threads dedicated for async IO operations (e.g. network activity) in native code. -#native.async.io.threads= - -## The number of threads dedicated for blocking IO operations (e.g. file access) in native code. -#native.max.blocking.threads= - -### Replication (QuestDB Enterprise Only) - -# Possible roles are "primary", "replica", or "none" -# primary - read/write node -# replica - read-only node, consuming data from PRIMARY -# none - replication is disabled -#replication.role=none - -## Object-store specific string. -## AWS S3 example: -## s3::bucket=${BUCKET_NAME};root=${DB_INSTANCE_NAME};region=${AWS_REGION};access_key_id=${AWS_ACCESS_KEY};secret_access_key=${AWS_SECRET_ACCESS_KEY}; -## Azure Blob example: -## azblob::endpoint=https://${STORE_ACCOUNT}.blob.core.windows.net;container={BLOB_CONTAINER};root=${DB_INSTANCE_NAME};account_name=${STORE_ACCOUNT};account_key=${STORE_KEY}; -## Filesystem: -## fs::root=/nfs/path/to/dir/final;atomic_write_dir=/nfs/path/to/dir/scratch; -#replication.object.store= - -## Limits the number of concurrent requests to the object store. -## Defaults to 128 -#replication.requests.max.concurrent=128 - -## Maximum number of times to retry a failed object store request before -## logging an error and reattempting later after a delay. -#replication.requests.retry.attempts=3 - -## Delay between the retry attempts (milliseconds) -#replication.requests.retry.interval=200 - -## The time window grouping multiple transactions into a replication batch (milliseconds). -## Smaller time windows use more network traffic. -## Larger time windows increase the replication latency. -## Works in conjunction with `replication.primary.throttle.non.data`. -#replication.primary.throttle.window.duration=10000 - -## Set to `false` to allow immediate replication of non-data transactions -## such as table creation, rename, drop, and uploading of any closed WAL segments. -## Only set to `true` if your application is highly sensitive to network overhead. -## In most cases, tweak `cairo.wal.segment.rollover.size` and -## `replication.primary.throttle.window.duration` instead. -#replication.primary.throttle.non.data=false - -## During the upload each table's transaction log is chunked into smaller sized -## "parts". Each part defaults to 5000 transactions. Before compression, each -## transaction requires 28 bytes, and the last transaction log part is re-uploaded -## in full each time for each data upload. -## If you are heavily network constraint you may want to lower this (to say 2000), -## but this would in turn put more pressure on the object store with a larger -## amount of tiny object uploads (values lower than 1000 are never recommended). -## Note: This setting only applies to newly created tables. -#replication.primary.sequencer.part.txn.count=5000 - -## Max number of threads used to perform file compression operations before -## uploading to the object store. The default value is calculated as half the -## number of CPU cores. -#replication.primary.compression.threads= - -## Zstd compression level. Defaults to 1. Valid values are from -7 to 22. -## Where -7 is fastest and least compressed, and 22 is most CPU and memory intensive -## and most compressed. -#replication.primary.compression.level=1 - -## Whether to calculate and include a checksum with every uploaded artifact. -## By default this is is done only for services (object store implementations) -## that don't provide this themselves (typically, the filesystem). -## The other options are "never" and "always". -#replication.primary.checksum=service-dependent - -## Each time the primary instance upload a new version of the index, -## it extends the ownership time lease of the object store, ensuring no other -## primary instance may be started in its place. -## The keepalive interval determines how long to wait after a period of inactivity -## before triggering a new empty ownership-extending upload. -## This setting also determines how long the database should when migrating ownership -## to a new instance. -## The wait is calculated as `3 x $replication.primary.keepalive.interval`. -#replication.primary.keepalive.interval=10s - -## Each upload for a given table will trigger an update of the index. -## If a database has many actively replicating tables, this could trigger -## overwriting the index path on the object store many times per second. -## This can be an issue with Google Cloud Storage which has a hard-limit of -## of 1000 requests per second, per path. -## As such, we default this to a 1s grouping time window for for GCS, and no -## grouping time window for other object store providers. -#replication.primary.index.upload.throttle.interval= - -## During the uploading process WAL column files may have unused trailing data. -## The default is to skip reading/compressing/uploading this data. -#replication.primary.upload.truncated=true - -## Polling rate of a replica instance to check for new changes (milliseconds). -#replication.replica.poll.interval=1000 - -## Network buffer size (byte count) used when downloading data from the object store. -#replication.requests.buffer.size=32768 - -## How often to produce a summary of the replication progress for each table -## in the logs. The default is to do this once a minute. -## You may disable the summary by setting this parameter to 0. -#replication.summary.interval=1m - -## Enable per-table Prometheus metrics on replication progress. -## These are enabled by default. -#replication.metrics.per.table=true - -## How many times (number of HTTP Prometheus polled scrapes) should per-table -## replication metrics continue to be published for dropped tables. -## The default of 10 is designed to ensure that transaction progress -## for dropped tables is not accidentally missed. -## Consider raising if you scrape each database instance from multiple Prometheus -## instances. -#replication.metrics.dropped.table.poll.count=10 - -## Number of parallel requests to cap at when uploading or downloading data for a given -## iteration. This cap is used to perform partial progress when there is a large amount -## of outstanding uploads/downloads. -## The `fast` version is used by default for good throughput when everything is working -## well, while `slow` is used after the uploads/downloads encounter issues to allow -## progress in case of flaky networking conditions. -# replication.requests.max.batch.size.fast=64 -# replication.requests.max.batch.size.slow=2 - -## When the replication logic uploads, it times out and tries again if requests take too -## long. Since the timeout is dependent on the upload artifact size, we use a base timeout -## and a minimum throughput. The base timeout is of 10 seconds. -## The additional throughput-derived timeout is calculated from the size of the artifact -## and expressed as bytes per second. -#replication.requests.base.timeout=10s -#replication.requests.min.throughput=262144 - -### OIDC (QuestDB Enterprise Only) - -# Enables/Disables OIDC authentication. Once enabled few other -# configuration options must also be set. -#acl.oidc.enabled=false - -# Required: OIDC provider hostname -#acl.oidc.host= - -# Required: OIDC provider port number. -#acl.oidc.port=443 - -# Optional: OIDC provider host TLS setting; TLS is enabled by default. -#acl.oidc.tls.enabled=true - -# Optional: Enables QuestDB to validate TLS configuration, such as -# validity of TLS certificates. If you are working with self-signed -# certificates that you would like QuestDB to trust, disable this validations. -# Validation is strongly recommended in production environments. -#acl.oidc.tls.validation.enabled=true - -# Optional: path to keystore that contains certificate information of the -# OIDC provider. This is not required if your OIDC provider uses public CAs -#acl.oidc.tls.keystore.path= - -# Optional: keystore password, if the keystore is password protected. -#acl.oidc.tls.keystore.password= - -# Optional: OIDC provider HTTP request timeout in milliseconds -#acl.oidc.http.timeout=30000 - -# Required: OIDC provider client name. Typically OIDC provider will -# require QuestDB instance to become a "client" of the OIDC server. This -# procedure requires a "client" named entity to be created on OIDC server. -# Copy the name of the client to this configuration option -#acl.oidc.client.id= - -# Optional: OIDC authorization endpoint; the default value should work for Ping Identity Platform -#acl.oidc.authorization.endpoint=/as/authorization.oauth2 - -# Optional: OAUTH2 token endpoint; the default value should work for Ping Identity Platform -#acl.oidc.token.endpoint=/as/token.oauth2 - -# Optional: OIDC user info endpoint; the default value should work for Ping Identity Platform -#acl.oidc.userinfo.endpoint=/idp/userinfo.openid - -# Required: OIDC provider must include group array into user info response. Typically -# this is a JSON response that contains list of claim objects. Group claim is -# the name of the claim in that JSON object that contains an array of user group -# names. -#acl.oidc.groups.claim=groups - -# Optional: QuestDB instance will cache tokens for the specified TTL (millis) period before -# they are revalidated again with OIDC provider. It is a performance related setting to avoid -# otherwise stateless QuestDB instance contacting OIDC provider too frequently. -# If set to 0, the cache is disabled completely. -#acl.oidc.cache.ttl=30000 - -# Optional: QuestDB instance will cache the OIDC provider's public keys which can be used to check -# a JWT token's validity. QuestDB will reload the public keys after this expiry time to pickup -# any changes. It is a performance related setting to avoid contacting the OIDC provider too frequently. -#acl.oidc.public.keys.expiry=120000 - -# Log timestamp is generated in UTC, however, it can be written out as -# timezone of your choice. Timezone name can be either explicit UTC offset, -# such as "+08:00", "-10:00", "-07:45", "UTC+05" or timezone name "EST", "CET" etc -# https://www.joda.org/joda-time/timezones.html. Please be aware that timezone database, -# that is included with QuestDB distribution is changing from one release to the next. -# This is not something that we change, rather Java releases include timezone database -# updates. -#log.timestamp.timezone=UTC - -# Timestamp locale. You can use this to have timestamp written out using localised month -# names and day of week names. For example, for Portuguese use 'pt' locale. -#log.timestamp.locale=en - -# Format pattern used to write out log timestamps -#log.timestamp.format=yyyy-MM-ddTHH:mm:ss.SSSUUUz - -#query.within.latest.by.optimisation.enabled diff --git a/ci/confs/default/server.conf b/ci/confs/default/server.conf deleted file mode 100644 index f96a35e..0000000 --- a/ci/confs/default/server.conf +++ /dev/null @@ -1,1395 +0,0 @@ -# Comment or set to false to allow QuestDB to start even in the presence of config errors. -config.validation.strict=true - -# toggle whether worker should stop on error -#shared.worker.haltOnError=false - -# Number of threads in Network shared thread pool. Network thread pool used for handling HTTP, TCP, UDP and Postgres connections unless dedicated thread pools are configured. -#shared.network.worker.count=2 - -# Comma-delimited list of CPU ids, one per thread specified in "shared.network.worker.count" used for Network thread pool. By default, threads have no CPU affinity -#shared.network.worker.affinity= - -# Number of threads in Query shared thread pool. Query thread pool used for handling parallel queries, like parallel filters and group by queries. -#shared.query.worker.count=2 - -# Comma-delimited list of CPU ids, one per thread specified in "shared.query.worker.count" used for Query thread pool. By default, threads have no CPU affinity -#shared.query.worker.affinity= - -# Number of threads in Write shared thread pool. Write pool threads are used for running WAL Apply work load to merge data from WAL files into the table -#shared.write.worker.count=2 - -# Comma-delimited list of CPU ids, one per thread specified in "shared.write.worker.count" used for Write thread pool. By default, threads have no CPU affinity -#shared.write.worker.affinity= - -# Default number of worker threads in Network, Query and Write shared pools. Single value to configure all three pools sizes. -#shared.worker.count=2 - -# RAM usage limit, as a percentage of total system RAM. A zero value does not -# set any limit. The default is 90. -#ram.usage.limit.percent=90 - -# RAM usage limit, in bytes. A zero value (the default) does not set any limit. -# If both this and ram.usage.limit.percent are non-zero, the lower limit takes precedence. -#ram.usage.limit.bytes=0 - -# Repeats compatible migrations from the specified version. The default setting of 426 allows to upgrade and downgrade QuestDB in the range of versions from 6.2.0 to 7.0.2. -# If set to -1 start time improves but downgrades to versions below 7.0.2 and subsequent upgrades can lead to data corruption and crashes. -#cairo.repeat.migration.from.version=426 - -################ HTTP settings ################## - -# enable HTTP server -http.enabled=false - -# IP address and port of HTTP server -#http.net.bind.to=0.0.0.0:9000 - -# Uncomment to enable HTTP Basic authentication -#http.user=admin -#http.password=quest - -# Maximum time interval for the HTTP server to accept connections greedily in a loop. -# The accept loop timeout is set in milliseconds. -#http.net.accept.loop.timeout=500 - -#http.net.connection.limit=256 -# Windows OS might have a limit on TCP backlog size. Typically Windows 10 has max of 200. This -# means that even if net.connection.limit is set over 200 it wont be possible to have this many -# concurrent connections. To overcome this limitation Windows has an unreliable hack, which you can -# exercise with this flag. Do only set it if you actively try to overcome limit you have already -# experienced. Read more about SOMAXCONN_HINT here https://docs.microsoft.com/en-us/windows/win32/api/winsock2/nf-winsock2-listen -#http.net.connection.hint=false - - -# Maximum HTTP connections that can be used for ILP ingestion using /write http endpoint. This limit must be lower or equal to http.net.connection.limit -# Not restricted by default. Database restart is NOT required when this setting is changed -#http.ilp.connection.limit=-1 - -# Maximum HTTP connections that can be used for queries using /query http endpoint. This limit must be lower or equal to http.net.connection.limit -# Not restricted by default. Database restart is NOT required when this setting is changed -#http.json.query.connection.limit=-1 - -# Maximum HTTP connections that can be used for export using /exp http endpoint. This limit must be lower or equal to http.net.connection.limit -# Restricted to number or CPUs or 25% of overall http connections, whichever is lower, by default. Database restart is NOT required when this setting is changed -#http.export.connection.limit=-1 - -# Idle HTTP connection timeout in milliseconds. -#http.net.connection.timeout=5m - -#Amount of time in milliseconds a connection can wait in the listen backlog queue before its refused. Connections will be aggressively removed from the backlog until the active connection limit is breached -#http.net.connection.queue.timeout=5000 - -# SO_SNDBUF value, -1 = OS default -#http.net.connection.sndbuf=2m - -# SO_RCVBUF value, -1 = OS default -#http.net.connection.rcvbuf=2m - -# size of receive buffer on application side -#http.receive.buffer.size=1m - -# initial size of the connection pool -#http.connection.pool.initial.capacity=4 - -# initial size of the string pool shared by HttpHeaderParser and HttpMultipartContentParser -#http.connection.string.pool.capacity=128 - -# HeaderParser buffer size in bytes -#http.multipart.header.buffer.size=512 - -# how long code accumulates incoming data chunks for column and delimiter analysis -#http.multipart.idle.spin.count=10000 - -#http.request.header.buffer.size=64k - -#http.worker.count=0 -#http.worker.affinity= -#http.worker.haltOnError=false - -# size of send data buffer -#http.send.buffer.size=2m - -# sets the clock to always return zero -#http.frozen.clock=false - -#http.allow.deflate.before.send=false - -# HTTP session timeout -#http.session.timeout=30m - -## When you using SSH tunnel you might want to configure -## QuestDB HTTP server to switch to HTTP/1.0 - -## Set HTTP protocol version to HTTP/1.0 -#http.version=HTTP/1.1 -## Set server keep alive to 'false'. This will make server disconnect client after -## completion of each request -#http.server.keep.alive=true - -## When in HTTP/1.0 mode keep alive values must be 0 -#http.keep-alive.timeout=5 -#http.keep-alive.max=10000 - -#http.static.public.directory=public - -#http.text.date.adapter.pool.capacity=16 -#http.text.json.cache.limit=16384 -#http.text.json.cache.size=8192 -#http.text.max.required.delimiter.stddev=0.1222d -#http.text.max.required.line.length.stddev=0.8 -#http.text.metadata.string.pool.capacity=128 -#http.text.roll.buffer.limit=8216576 -#http.text.roll.buffer.size=1024 -#http.text.analysis.max.lines=1000 -#http.text.lexer.string.pool.capacity=64 -#http.text.timestamp.adapter.pool.capacity=64 -#http.text.utf8.sink.size=4096 - -#http.json.query.connection.check.frequency=1000000 - -# enables the query cache -#http.query.cache.enabled=true - -# sets the number of blocks for the query cache. Cache capacity is number_of_blocks * number_of_rows. -#http.query.cache.block.count= 8 * worker_count - -# sets the number of rows for the query cache. Cache capacity is number_of_blocks * number_of_rows. -#http.query.cache.row.count= 2 * worker_count - -# sets the /settings endpoint readonly -#http.settings.readonly=false - -#http.security.readonly=false -#http.security.max.response.rows=Long.MAX_VALUE - -# Context path for the Web Console. If other REST services remain on the -# default context paths they will move to the same context path as the Web Console. -# Exception is ILP HTTP services, which are not used by the Web Console. They will -# remain on their default context paths. When the default context paths are changed, -# moving the Web Console will not affect the configured paths. QuestDB will create -# a copy of those services on the paths used by the Web Console so the outcome is -# both the Web Console and the custom service are operational. -#http.context.web.console=/ - -# Context path of the file import service -#http.context.import=/imp -# This service is used by the import UI in the Web Console -#http.context.table.status=/chk - -# Context path of the SQL result CSV export service -#http.context.export=/exp - -# This service provides server-side settings to the Web Console -#http.context.settings=/settings - -# SQL execution service -#http.context.execute=/exec - -# Web Console specific service -#http.context.warnings=/warnings - -# ILP HTTP Services. These are not used by the Web Console -#http.context.ilp=/write,/api/v2/write -#http.context.ilp.ping=/ping - -# Custom HTTP redirect service. All redirects are 301 - Moved permanently -#http.redirect.count=1 -#http.redirect.1=/ -> /index.html - -# circuit breaker is a mechanism that interrupts query execution -# at present queries are interrupted when remote client disconnects or when execution takes too long -# and times out - -# circuit breaker is designed to be invoke continuously in a tight loop -# the throttle is a number of pin cycles before abort conditions are tested -#circuit.breaker.throttle=2000000 - -# buffer used by I/O dispatchers and circuit breakers to check the socket state, please do not change this value -# the check reads \r\n from the input stream and discards it since some HTTP clients send this as a keep alive in between requests -#net.test.connection.buffer.size=64 - -# max execution time for read-only query in seconds -# "insert" type of queries are not aborted unless they -# it is "insert as select", where select takes long time before producing rows for the insert -query.timeout=5s - -## HTTP MIN settings -## -## Use this port to health check QuestDB instance when it isn't desired to log these health check requests. This is sort of /dev/null for monitoring - -#http.min.enabled=true -#http.min.net.bind.to=0.0.0.0:9003 - -# When enabled, health check will return HTTP 500 if there were any unhandled errors since QuestDB instance start. -#http.pessimistic.health.check.enabled=false - -# Maximum time interval for the HTTP MIN server to accept connections greedily in a loop. -# The accept loop timeout is set in milliseconds. -#http.min.net.accept.loop.timeout=500 - -################ Cairo settings ################## - -# directory for storing db tables and metadata. this directory is inside the server root directory provided at startup -#cairo.root=db - -# how changes to table are flushed to disk upon commit - default: nosync. Choices: nosync, async (flush call schedules update, returns immediately), sync (waits for flush to complete) -#cairo.commit.mode=nosync - -# The amount of time server is allowed to be compounding transaction before physical commit is forced. -# Compounding of the transactions improves system's through but hurts latency. Reduce this value to -# reduce latency of data visibility. -#cairo.commit.latency=30s - -# number of types table creation or insertion will be attempted -#cairo.create.as.select.retry.count=5 - -# comma separated list of volume definitions, volume_alias -> absolute_path_to_existing_directory. -# volume alias can then be used in create table statement with IN VOLUME clause -#cairo.volumes= by default IN VOLUME is switched off, no volume definitions. - -# type of map uses. Options: 1. fast (speed at the expense of storage. this is the default option) 2. compact -#cairo.default.map.type=fast - -# when true, symbol values will be cached on Java heap -#cairo.default.symbol.cache.flag=false - -# when column type is SYMBOL this parameter specifies approximate capacity for symbol map. -# It should be equal to number of unique symbol values stored in the table and getting this -# value badly wrong will cause performance degradation. Must be power of 2 -#cairo.default.symbol.capacity=256 - -# number of attempts to open files -#cairo.file.operation.retry.count=30 - -# when DB is running in sync/async mode, how many 'steps' IDGenerators will pre-allocate and synchronize sync to disk -#cairo.id.generator.batch.step=512 - -# how often the writer maintenance job gets run -#cairo.idle.check.interval=5m - -# defines the number of latest partitions to keep open when returning a reader to the reader pool -#cairo.inactive.reader.max.open.partitions=128 - -# defines frequency in milliseconds with which the reader pool checks for inactive readers. -#cairo.inactive.reader.ttl=2m - -# defines frequency in milliseconds with which the writer pool checks for inactive readers. -#cairo.inactive.writer.ttl=10m - -# when true (default), TTL enforcement uses wall clock time to prevent accidental data loss -# when future timestamps are inserted. When false, TTL uses only the max timestamp in the table. -#cairo.ttl.use.wall.clock=true - -# approximation of number of rows for single index key, must be power of 2 -#cairo.index.value.block.size=256 - -# number of attempts to open swap file -#cairo.max.swap.file.count=30 - -# file permission for new directories -#cairo.mkdir.mode=509 - -# Time to wait before retrying writing into a table after a memory limit failure -#cairo.write.back.off.timeout.on.mem.pressure=4s - -# maximum file name length in chars. Affects maximum table name length and maximum column name length -#cairo.max.file.name.length=127 - -# minimum number of rows before allowing use of parallel indexation -#cairo.parallel.index.threshold=100000 - -# number of segments in the TableReader pool; each segment holds up to 32 readers -#cairo.reader.pool.max.segments=10 - -# timeout in milliseconds when attempting to get atomic memory snapshots, e.g. in BitmapIndexReaders -#cairo.spin.lock.timeout=1s - -# sets size of the CharacterStore -#cairo.character.store.capacity=1024 - -# Sets size of the CharacterSequence pool -#cairo.character.store.sequence.pool.capacity=64 - -# sets size of the Column pool in the SqlCompiler -#cairo.column.pool.capacity=4096 - -# size of the ExpressionNode pool in SqlCompiler -#cairo.expression.pool.capacity=8192 - -# load factor for all FastMaps -#cairo.fast.map.load.factor=0.7 - -# size of the JoinContext pool in SqlCompiler -#cairo.sql.join.context.pool.capacity=64 - -# size of FloatingSequence pool in GenericLexer -#cairo.lexer.pool.capacity=2048 - -# sets the key capacity in FastMap and CompactMap -#cairo.sql.map.key.capacity=2097152 - -# sets the key capacity in FastMap and CompactMap used in certain queries, e.g. SAMPLE BY -#cairo.sql.small.map.key.capacity=32 - -# number of map resizes in FastMap and CompactMap before a resource limit exception is thrown, each resize doubles the previous size -#cairo.sql.map.max.resizes=2^31 - -# memory page size for FastMap and CompactMap -#cairo.sql.map.page.size=4m - -# memory page size in FastMap and CompactMap used in certain queries, e.g. SAMPLE BY -#cairo.sql.small.map.page.size=32k - -# memory max pages for CompactMap -#cairo.sql.map.max.pages=2^31 - -# sets the size of the QueryModel pool in the SqlCompiler -#cairo.model.pool.capacity=1024 - -# sets the maximum allowed negative value used in LIMIT clause in queries with filters -#cairo.sql.max.negative.limit=10000 - -# sets the memory page size for storing keys in LongTreeChain -#cairo.sql.sort.key.page.size=128k - -# max number of pages for storing keys in LongTreeChain before a resource limit exception is thrown -# cairo.sql.sort.key.max.pages=2^31 - -# sets the memory page size and max pages for storing values in LongTreeChain -#cairo.sql.sort.light.value.page.size=128k -#cairo.sql.sort.light.value.max.pages=2^31 - -# sets the memory page size and max pages of the slave chain in full hash joins -#cairo.sql.hash.join.value.page.size=16777216 -#cairo.sql.hash.join.value.max.pages=2^31 - -# sets the initial capacity for row id list used for latest by -#cairo.sql.latest.by.row.count=1000 - -# sets the memory page size and max pages of the slave chain in light hash joins -#cairo.sql.hash.join.light.value.page.size=128k -#cairo.sql.hash.join.light.value.max.pages=2^31 - -# number of rows to scan linearly before starting binary search in ASOF JOIN queries with no additional keys -#cairo.sql.asof.join.lookahead=10 - -# sets memory page size and max pages of file storing values in SortedRecordCursorFactory -#cairo.sql.sort.value.page.size=16777216 -#cairo.sql.sort.value.max.pages=2^31 - -# latch await timeout in nanoseconds for stealing indexing work from other threads -#cairo.work.steal.timeout.nanos=10000 - -# whether parallel indexation is allowed. Works in conjunction with cairo.parallel.index.threshold -#cairo.parallel.indexing.enabled=true - -# memory page size for JoinMetadata file -#cairo.sql.join.metadata.page.size=16384 - -# number of map resizes in JoinMetadata before a resource limit exception is thrown, each resize doubles the previous size -#cairo.sql.join.metadata.max.resizes=2^31 - -# size of PivotColumn pool in SqlParser -#cairo.sql.pivot.column.pool.capacity=64 - -# maximum number of columns PIVOT can produce (FOR value combinations × aggregates) -#cairo.sql.pivot.max.produced.columns=5000 - -# size of WindowColumn pool in SqlParser -#cairo.sql.window.column.pool.capacity=64 - -# sets the memory page size and max number of pages for records in window function -#cairo.sql.window.store.page.size=1m -#cairo.sql.window.store.max.pages=2^31 - -# sets the memory page size and max number of pages for row ids in window function -#cairo.sql.window.rowid.page.size=512k -#cairo.sql.window.rowid.max.pages=2^31 - -# sets the memory page size and max number of pages for keys in window function -#cairo.sql.window.tree.page.size=512k -#cairo.sql.window.tree.max.pages=2^31 - -# sets initial size of per-partition window function range frame buffer -#cairo.sql.window.initial.range.buffer.size=32 - -# batch size of non-atomic inserts for CREATE TABLE AS SELECT statements -#cairo.sql.create.table.model.batch.size=1000000 - -# Size of the pool for model objects, that underpin the "create table" parser. -# It is aimed at reducing allocations and it a performance setting. The -# number is aligned to the max concurrent "create table" requests the system -# will ever receive. If system receives more requests that this, it will just -# allocate more object and free them after use. -#cairo.create.table.column.model.pool.capacity=16 - -# size of RenameTableModel pool in SqlParser -#cairo.sql.rename.table.model.pool.capacity=16 - -# size of WithClauseModel pool in SqlParser -#cairo.sql.with.clause.model.pool.capacity=128 - -# size of CompileModel pool in SqlParser -#cairo.sql.compile.view.model.pool.capacity=8 - -# initial size of view lexer pool in SqlParser, used to parse SELECT statements of view definitions -# the max number of views used in a single query determines how many view lexers should be in the pool -#cairo.sql.view.lexer.pool.capacity=8 - -# size of InsertModel pool in SqlParser -#cairo.sql.insert.model.pool.capacity=64 - -# batch size of non-atomic inserts for INSERT INTO SELECT statements -#cairo.sql.insert.model.batch.size=1000000 - -# enables parallel GROUP BY execution; by default, parallel GROUP BY requires at least 4 shared worker threads to take place -#cairo.sql.parallel.groupby.enabled=true - -# merge queue capacity for parallel GROUP BY; used for parallel tasks that merge shard hash tables -#cairo.sql.parallel.groupby.merge.shard.queue.capacity= - -# threshold for parallel GROUP BY to shard the hash table holding the aggregates -#cairo.sql.parallel.groupby.sharding.threshold=10000 - -# enables statistics-based hash table pre-sizing in parallel GROUP BY -#cairo.sql.parallel.groupby.presize.enabled=true - -# maximum allowed hash table size for parallel GROUP BY hash table pre-sizing -#cairo.sql.parallel.groupby.presize.max.capacity=100000000 - -# maximum allowed heap size for parallel GROUP BY hash table pre-sizing -#cairo.sql.parallel.groupby.presize.max.heap.size=1G - -# threshold for parallel ORDER BY + LIMIT execution on sharded GROUP BY hash table -#cairo.sql.parallel.groupby.topk.threshold=5000000 - -# queue capacity for parallel ORDER BY + LIMIT applied to sharded GROUP BY -#cairo.sql.parallel.groupby.topk.queue.capacity= - -# threshold for in-flight tasks for disabling work stealing during parallel SQL execution -# when the number of shared workers is less than 4x of this setting, work stealing is always enabled -#cairo.sql.parallel.work.stealing.threshold=16 - -# spin timeout in nanoseconds for adaptive work stealing strategy -# controls how long the query thread waits for worker threads to pick up tasks before stealing work back -#cairo.sql.parallel.work.stealing.spin.timeout=50000 - -# enables parallel read_parquet() SQL function execution; by default, parallel read_parquet() requires at least 4 shared worker threads to take place -#cairo.sql.parallel.read.parquet.enabled=true - -# capacity for Parquet page frame cache; larger values may lead to better ORDER BY and some other -# clauses performance at the cost of memory overhead -#cairo.sql.parquet.frame.cache.capacity=3 - -# default size for memory buffers in GROUP BY function native memory allocator -#cairo.sql.groupby.allocator.default.chunk.size=128K - -# maximum allowed native memory allocation for GROUP BY functions -#cairo.sql.groupby.allocator.max.chunk.size=4G - -# threshold in bytes for switching from single memory buffer hash table (unordered) to a hash table with separate heap for entries (ordered) -#cairo.sql.unordered.map.max.entry.size=32 - -## prevents stack overflow errors when evaluating complex nested SQLs -## the value is an approximate number of nested SELECT clauses. -#cairo.sql.window.max.recursion=128 - -## pre-sizes the internal data structure that stores active query executions -## the value is chosen automatically based on the number of threads in the shared worker pool -#cairo.sql.query.registry.pool.size= - -## window function buffer size in record counts -## pre-sizes buffer for every windows function execution to contain window records -#cairo.sql.analytic.initial.range.buffer.size=32 - -## enables quick and radix sort in order by, when applicable -#cairo.sql.orderby.sort.enabled=true - -## defines number of rows to use radix sort in order by -#cairo.sql.orderby.radix.sort.threshold=600 - -## enables the column alias to be generated from the expression -#cairo.sql.column.alias.expression.enabled=true - -## maximum length of generated column aliases -#cairo.sql.column.alias.generated.max.size=64 - -## initial capacity of string pool for preferences store and parser -#cairo.preferences.string.pool.capacity=64 - -## Flag to enable or disable symbol capacity auto-scaling. Auto-scaling means resizing -## symbol table data structures as the number of symbols in the table grows. Optimal sizing of -## these data structures ensures optimal ingres performance. -## -## By default, the auto-scaling is enabled. This is optimal. You may want to disable auto-scaling in case -## something goes wrong. -## -## Database restart is NOT required when this setting is changed, but `reload_config()` SQL should be executed. -#cairo.auto.scale.symbol.capacity=true - -## Symbol occupancy threshold after which symbol capacity is doubled. For example -## threshold of 0.8 means that occupancy have to reach 80% of capacity before capacity is increased -#cairo.auto.scale.symbol.capacity.threshold=0.8 - -#### SQL COPY - -# size of CopyModel pool in SqlParser -#cairo.sql.copy.model.pool.capacity=32 - -# size of buffer used when copying tables -#cairo.sql.copy.buffer.size=2m - -# name of file with user's set of date and timestamp formats -#cairo.sql.copy.formats.file=/text_loader.json - -# input root directory, where COPY command and read_parquet() function read files from -# relative paths are resolved against the server root directory -cairo.sql.copy.root=import - -# export root directory, where COPY .. to command write files to -# relative paths are resolved against the server root directory -#cairo.sql.copy.export.root=export - -# input work directory, where temporary import files are created, by default it's located in tmp directory inside the server root directory -#cairo.sql.copy.work.root=null - -# default max size of intermediate import file index chunk (100MB). Import shouldn't use more memory than worker_count * this . -#cairo.sql.copy.max.index.chunk.size=100M - -# Capacity of the internal queue used to split parallel copy SQL command into subtasks and execute them across shared worker threads. -# The default configuration should be suitable for importing files of any size. -#cairo.sql.copy.queue.capacity=32 - -# Capacity of the internal queue used to execute copy export SQL command. -#cairo.sql.copy.export.queue.capacity=32 - -# Frequency of logging progress when exporting data using COPY .. TO command to export to parquet format. 0 or negative value disables the logging. -# Database restart is NOT required when this setting is changed -#cairo.parquet.export.copy.report.frequency.lines=50000 - -# Parquet version to use when exporting data using COPY .. TO command. Valid values: 1 (PARQUET_1_0), 2 (PARQUET_2_LATEST) -#cairo.parquet.export.version=1 - -# Enable statistics collection in Parquet files when exporting data using COPY .. TO command -#cairo.parquet.export.statistics.enabled=true - -# Enable raw array encoding for repeated fields in Parquet files when exporting data using COPY .. TO command -#cairo.parquet.export.raw.array.encoding.enabled=false - -# Compression codec to use when exporting data using COPY .. TO command. Valid values: -# UNCOMPRESSED, SNAPPY, GZIP, BROTLI, ZSTD, LZ4_RAW -#cairo.parquet.export.compression.codec=ZSTD - -# Compression level for GZIP (0-9), BROTLI (0-11), or ZSTD (1-22) codecs when exporting data using COPY .. TO command -#cairo.parquet.export.compression.level=9 - -# Row group size in rows when exporting data using COPY .. TO command. 0 uses the default (512*512 rows) -# Database restart is NOT required when this setting is changed -#cairo.parquet.export.row.group.size=0 - -# Data page size in bytes when exporting data using COPY .. TO command. 0 uses the default (1024*1024 bytes) -# Database restart is NOT required when this setting is changed -#cairo.parquet.export.data.page.size=0 - -# Maximum time to wait for export to complete when using /exp HTTP endpoint. 0 means no timeout. -#http.export.timeout=300s - -# Parquet version to use for partition encoder (for storing partitions in parquet format). Valid values: 1 (PARQUET_1_0), 2 (PARQUET_2_LATEST) -#cairo.partition.encoder.parquet.version=1 - -# Enable statistics collection in Parquet files for partition encoder -#cairo.partition.encoder.parquet.statistics.enabled=true - -# Enable raw array encoding for repeated fields in Parquet files for partition encoder -#cairo.partition.encoder.parquet.raw.array.encoding.enabled=false - -# Compression codec to use for partition encoder. Valid values: -# UNCOMPRESSED, SNAPPY, GZIP, BROTLI, ZSTD, LZ4_RAW -#cairo.partition.encoder.parquet.compression.codec=ZSTD - -# Compression level for GZIP (0-9), BROTLI (0-11), or ZSTD (1-22) codecs for partition encoder -#cairo.partition.encoder.parquet.compression.level=9 - -# Row group size in rows for partition encoder -#cairo.partition.encoder.parquet.row.group.size=100000 - -# Data page size in bytes for partition encoder -#cairo.partition.encoder.parquet.data.page.size=1048576 - -# Number of days to retain records in import log table (sys.parallel_text_import_log). Old records get deleted on each import and server restart. -#cairo.sql.copy.log.retention.days=3 - -# output root directory for backups -#cairo.sql.backup.root=null - -# date format for backup directory -#cairo.sql.backup.dir.datetime.format=yyyy-MM-dd - -# name of temp directory used during backup -#cairo.sql.backup.dir.tmp.name=tmp - -# permission used when creating backup directories -#cairo.sql.backup.mkdir.mode=509 - -# suffix of the partition directory in detached root to indicate it is ready to be attached -#cairo.attach.partition.suffix=.attachable - -# Use file system "copy" operation instead of "hard link" when attaching partition from detached root. Set to true if detached root is on a different drive. -#cairo.attach.partition.copy=false - -# file permission used when creating detached directories -#cairo.detached.mkdir.mode=509 - -# sample by index query page size - max values returned in single scan -# 0 means to use symbol block capacity -# cairo.sql.sampleby.page.size=0 - -# sample by default alignment -# if true, sample by will default to SAMPLE BY (FILL) ALIGN TO CALENDAR -# if false, sample by will default to SAMPLE BY (FILL) ALIGN TO FIRST OBSERVATION -# cairo.sql.sampleby.default.alignment.calendar=true - -# sets the minimum number of rows in page frames used in SQL queries -#cairo.sql.page.frame.min.rows=100000 - -# sets the maximum number of rows in page frames used in SQL queries -#cairo.sql.page.frame.max.rows=1000000 - -# sets the minimum number of rows in small page frames used in SQL queries, primarily for window joins -#cairo.sql.small.page.frame.min.rows=10000 - -# sets the maximum number of rows in small page frames used in SQL queries, primarily for window joins -#cairo.sql.small.page.frame.max.rows=100000 - -# sets the memory page size and max number of pages for memory used by rnd functions -# currently rnd_str() and rnd_symbol(), this could extend to other rnd functions in the future -#cairo.rnd.memory.page.size=8K -#cairo.rnd.memory.max.pages=128 - -# max length (in chars) of buffer used to store result of SQL functions, such as replace() or lpad() -#cairo.sql.string.function.buffer.max.size=1048576 - -# SQL JIT compiler mode. Options: -# 1. on (enable JIT and use vector instructions when possible; default value) -# 2. scalar (enable JIT and use scalar instructions only) -# 3. off (disable JIT) -#cairo.sql.jit.mode=on - -# sets the memory page size and max pages for storing IR for JIT compilation -#cairo.sql.jit.ir.memory.page.size=8K -#cairo.sql.jit.ir.memory.max.pages=8 - -# sets the memory page size and max pages for storing bind variable values for JIT compiled filter -#cairo.sql.jit.bind.vars.memory.page.size=4K -#cairo.sql.jit.bind.vars.memory.max.pages=8 - -# sets debug flag for JIT compilation; when enabled, assembly will be printed into stdout -#cairo.sql.jit.debug.enabled=false - -# Controls the maximum allowed IN list length before the JIT compiler will fall back to Java code -#cairo.sql.jit.max.in.list.size.threshold=10 - -#cairo.date.locale=en - -# Maximum number of uncommitted rows in TCP ILP -#cairo.max.uncommitted.rows=500000 - -# Minimum size of in-memory buffer in milliseconds. The buffer is allocated dynamically through analysing -# the shape of the incoming data, and o3MinLag is the lower limit -#cairo.o3.min.lag=1s - -# Maximum size of in-memory buffer in milliseconds. The buffer is allocated dynamically through analysing -# the shape of the incoming data, and o3MaxLag is the upper limit -#cairo.o3.max.lag=600000 - -# Memory page size per column for O3 operations. Please be aware O3 will use 2x of this RAM per column -#cairo.o3.column.memory.size=8M - -# Memory page size per column for O3 operations on System tables only -#cairo.system.o3.column.memory.size=256k - -# Number of partition expected on average, initial value for purge allocation job, extended in runtime automatically -#cairo.o3.partition.purge.list.initial.capacity=1 - -# mmap sliding page size that TableWriter uses to append data for each column -#cairo.writer.data.append.page.size=16M - -# mmap page size for mapping small files, such as _txn, _todo and _meta -# the default value is OS page size (4k Linux, 64K windows, 16k OSX M1) -# if you override this value it will be rounded to the nearest (greater) multiple of OS page size -#cairo.writer.misc.append.page.size=4k - -# mmap page size for appending index key data; key data is number of distinct symbol values times 4 bytes -#cairo.writer.data.index.key.append.page.size=512k - -# mmap page size for appending value data; value data are rowids, e.g. number of rows in partition times 8 bytes -#cairo.writer.data.index.value.append.page.size=16M - -# mmap sliding page size that TableWriter uses to append data for each column specifically for System tables -#cairo.system.writer.data.append.page.size=256k - -# File allocation page min size for symbol table files -#cairo.symbol.table.min.allocation.page.size=4k - -# File allocation page max size for symbol table files -#cairo.symbol.table.max.allocation.page.size=8M - -# Maximum wait timeout in milliseconds for ALTER TABLE SQL statement run via REST and PG Wire interfaces when statement execution is ASYNCHRONOUS -#cairo.writer.alter.busy.wait.timeout=500ms - -# Row count to check writer command queue after on busy writing (e.g. tick after X rows written) -#cairo.writer.tick.rows.count=1024 - -# Maximum writer ALTER TABLE and replication command capacity. Shared between all the tables -#cairo.writer.command.queue.capacity=32 - -# Sets flag to enable io_uring interface for certain disk I/O operations on newer Linux kernels (5.12+). -#cairo.iouring.enabled=true - -# Minimum O3 partition prefix size for which O3 partition split happens to avoid copying the large prefix -#cairo.o3.partition.split.min.size=50M - -# The number of O3 partition splits allowed for the last partitions. If the number of splits grows above this value, the splits will be squashed -#cairo.o3.last.partition.max.splits=20 - -################ Parallel SQL execution ################ - -# Sets flag to enable parallel SQL filter execution. JIT compilation takes place only when this setting is enabled. -#cairo.sql.parallel.filter.enabled=true - -# Sets the threshold for column pre-touch to be run as a part of the parallel SQL filter execution. The threshold defines ratio between the numbers of scanned and filtered rows. -#cairo.sql.parallel.filter.pretouch.threshold=0.05 - -# Sets the upper limit on the number of in-flight tasks for parallel query workers published by filter queries with LIMIT. -#cairo.sql.parallel.filter.dispatch.limit= - -# Sets flag to enable parallel ORDER BY + LIMIT SQL execution. -#cairo.sql.parallel.topk.enabled=true - -# Sets flag to enable parallel WINDOW JOIN SQL execution. -#cairo.sql.parallel.window.join.enabled=true - -# Shard reduce queue contention between SQL statements that are executed concurrently. -#cairo.page.frame.shard.count=4 - -# Reduce queue is used for data processing and should be large enough to supply tasks for worker threads (shared worked pool). -#cairo.page.frame.reduce.queue.capacity= - -# Reduce queue is used for vectorized data processing and should be large enough to supply tasks for worker threads (shared worked pool). -#cairo.vector.aggregate.queue.capacity= - -# Initial row ID list capacity for each slot of the "reduce" queue. Larger values reduce memory allocation rate, but increase RSS size. -#cairo.page.frame.rowid.list.capacity=256 - -# Initial column list capacity for each slot of the "reduce" queue. Used by JIT-compiled filters. -#cairo.page.frame.column.list.capacity=16 - -# Initial object pool capacity for local "reduce" tasks. These tasks are used to avoid blocking query execution when the "reduce" queue is full. -#cairo.page.frame.task.pool.capacity=4 - -################ LINE settings ###################### - -#line.default.partition.by=DAY - -# Enable / Disable automatic creation of new columns in existing tables via ILP. When set to false overrides value of line.auto.create.new.tables to false -#line.auto.create.new.columns=true - -# Enable / Disable automatic creation of new tables via ILP. -#line.auto.create.new.tables=true - -# Enable / Disable printing problematic ILP messages in case of errors. -#line.log.message.on.error=true - -################ LINE UDP settings ################## - -#line.udp.bind.to=0.0.0.0:9009 -#line.udp.join=232.1.2.3 -#line.udp.commit.rate=1000000 -#line.udp.msg.buffer.size=2048 -#line.udp.msg.count=10000 -#line.udp.receive.buffer.size=8m -line.udp.enabled=true -#line.udp.own.thread.affinity=-1 -#line.udp.own.thread=false -#line.udp.unicast=false -#line.udp.commit.mode=nosync -#line.udp.timestamp=n - -######################### LINE TCP settings ############################### - -#line.tcp.enabled=true -#line.tcp.net.bind.to=0.0.0.0:9009 -#line.tcp.net.connection.limit=256 - -# Windows OS might have a limit on TCP backlog size. Typically Windows 10 has max of 200. This -# means that even if net.connection.limit is set over 200 it wont be possible to have this many -# concurrent connections. To overcome this limitation Windows has an unreliable hack, which you can -# exercise with this flag. Do only set it if you actively try to overcome limit you have already -# experienced. Read more about SOMAXCONN_HINT here https://docs.microsoft.com/en-us/windows/win32/api/winsock2/nf-winsock2-listen -#line.tcp.net.connection.hint=false - -# Idle TCP connection timeout in milliseconds. 0 means there is no timeout. -#line.tcp.net.connection.timeout=0 - -# Amount of time in milliseconds a connection can wait in the listen backlog queue before its refused. Connections will be aggressively removed from the backlog until the active connection limit is breached -#line.tcp.net.connection.queue.timeout=5000 - -# Maximum time interval for the ILP TCP endpoint to accept connections greedily in a loop. -# The accept loop timeout is set in milliseconds. -#line.tcp.net.accept.loop.timeout=500 - -# SO_RCVBUF value, -1 = OS default -#line.tcp.net.connection.rcvbuf=-1 - -#line.tcp.connection.pool.capacity=64 -#line.tcp.timestamp=n - -# TCP message buffer size -#line.tcp.msg.buffer.size=2048 - -# Max measurement size -#line.tcp.max.measurement.size=2048 - -# Max receive buffer size -#line.tcp.max.recv.buffer.size=1073741824 - -# Size of the queue between the IO jobs and the writer jobs, each queue entry represents a measurement -#line.tcp.writer.queue.capacity=128 - -# IO and writer job worker pool settings, 0 indicates the shared pool should be used -#line.tcp.writer.worker.count=0 -#line.tcp.writer.worker.affinity= -#line.tcp.writer.worker.yield.threshold=10 -#line.tcp.writer.worker.nap.threshold=7000 -#line.tcp.writer.worker.sleep.threshold=10000 -#line.tcp.writer.halt.on.error=false - -#line.tcp.io.worker.count=0 -#line.tcp.io.worker.affinity= -#line.tcp.io.worker.yield.threshold=10 -#line.tcp.io.worker.nap.threshold=7000 -#line.tcp.io.worker.sleep.threshold=10000 -#line.tcp.io.halt.on.error=false - -# Sets flag to disconnect TCP connection that sends malformed messages. -#line.tcp.disconnect.on.error=true - -# Commit lag fraction. Used to calculate commit interval for the table according to the following formula: -# commit_interval = commit_lag ∗ fraction -# The calculated commit interval defines how long uncommitted data will need to remain uncommitted. -#line.tcp.commit.interval.fraction=0.5 -# Default commit interval in milliseconds. Used when o3MinLag is set to 0. -#line.tcp.commit.interval.default=2000 - -# Maximum amount of time in between maintenance jobs in milliseconds, these will commit uncommitted data -#line.tcp.maintenance.job.interval=1000 -# Minimum amount of idle time before a table writer is released in milliseconds -#line.tcp.min.idle.ms.before.writer.release=500 - - -#line.tcp.symbol.cache.wait.before.reload=500ms - -######################### LINE HTTP settings ############################### - -#line.http.enabled=true - -#line.http.max.recv.buffer.size=1073741824 - -#line.http.ping.version=v2.7.4 - -################ PG Wire settings ################## - -#pg.enabled=true -#pg.net.bind.to=0.0.0.0:8812 -#pg.net.connection.limit=64 -# Windows OS might have a limit on TCP backlog size. Typically Windows 10 has max of 200. This -# means that even if active.connection.limit is set over 200 it wont be possible to have this many -# concurrent connections. To overcome this limitation Windows has an unreliable hack, which you can -# exercise with this flag. Do only set it if you actively try to overcome limit you have already -# experienced. Read more about SOMAXCONN_HINT here https://docs.microsoft.com/en-us/windows/win32/api/winsock2/nf-winsock2-listen -#pg.net.connection.hint=false - -# Connection idle timeout in milliseconds. Connections are closed by the server when this timeout lapses. -#pg.net.connection.timeout=300000 - -# Amount of time in milliseconds a connection can wait in the listen backlog queue before its refused. Connections will be aggressively removed from the backlog until the active connection limit is breached. -#pg.net.connection.queue.timeout=5m - -# Maximum time interval for the PG Wire server to accept connections greedily in a loop. -# The accept loop timeout is set in milliseconds. -#pg.net.accept.loop.timeout=500 - -# SO_RCVBUF value, -1 = OS default -#pg.net.connection.rcvbuf=-1 - -# SO_SNDBUF value, -1 = OS default -#pg.net.connection.sndbuf=-1 - -#pg.legacy.mode.enabled=false -#pg.character.store.capacity=4096 -#pg.character.store.pool.capacity=64 -#pg.connection.pool.capacity=64 -#pg.password=quest -#pg.user=admin -# Enables read-only mode for the pg wire protocol. In this mode data mutation queries are rejected. -#pg.security.readonly=false -#pg.readonly.password=quest -#pg.readonly.user=user -# Enables separate read-only user for the pg wire server. Data mutation queries are rejected for all connections opened by this user. -#pg.readonly.user.enabled=false -# enables select query cache -#pg.select.cache.enabled=true -# sets the number of blocks for the select query cache. Cache capacity is number_of_blocks * number_of_rows. -#pg.select.cache.block.count= 8 * worker_count -# sets the number of rows for the select query cache. Cache capacity is number_of_blocks * number_of_rows. -#pg.select.cache.row.count= 2 * worker_count -# enables insert query cache -#pg.insert.cache.enabled=true -# sets the number of blocks for the insert query cache. Cache capacity is number_of_blocks * number_of_rows. -#pg.insert.cache.block.count=4 -# sets the number of rows for the insert query cache. Cache capacity is number_of_blocks * number_of_rows. -#pg.insert.cache.row.count=4 -#pg.max.blob.size.on.query=512k -#pg.recv.buffer.size=1M -#pg.net.connection.rcvbuf=-1 -#pg.send.buffer.size=1M -#pg.net.connection.sndbuf=-1 -#pg.date.locale=en -#pg.worker.count=2 -#pg.worker.affinity=-1,-1; -#pg.halt.on.error=false -#pg.daemon.pool=true -#pg.binary.param.count.capacity=2 -# maximum number of prepared statements a single client can create at a time. This is to prevent clients from creating -# too many prepared statements and exhausting server resources. -#pg.named.statement.limit=10000 - -# if you are using insert batches of over 64 rows, you should increase this value to avoid memory resizes that -# might slow down inserts. Be careful though as this allocates objects on JavaHeap. Setting this value too large -# might kill the GC and server will be unresponsive on startup. -#pg.pipeline.capacity=64 - -################ Materialized View settings ################## - -# Enables SQL support and refresh job for materialized views. -#cairo.mat.view.enabled=true - -# When disabled, SQL executed by materialized view refresh job always runs single-threaded. -#cairo.mat.view.parallel.sql.enabled=true - -# Desired number of base table rows to be scanned by one query during materialized view refresh. -#cairo.mat.view.rows.per.query.estimate=1000000 - -# Maximum time interval used for a single step of materialized view refresh. For larger intervals single SAMPLE BY interval will be used as the step. -#cairo.mat.view.max.refresh.step=31536000000000us - -# Maximum number of times QuestDB will retry a materialized view refresh after a recoverable error. -#cairo.mat.view.max.refresh.retries=10 - -# Timeout for next refresh attempts after an out-of-memory error in a materialized view refresh, in milliseconds. -#cairo.mat.view.refresh.oom.retry.timeout=200 - -# Maximum number of rows inserted by the refresh job in a single transaction. -#cairo.mat.view.insert.as.select.batch.size=1000000 - -# Maximum number of time intervals from WAL data transactions cached per materialized view. -#cairo.mat.view.max.refresh.intervals=100 - -# Interval for periodical refresh intervals caching for materialized views. -#cairo.mat.view.refresh.intervals.update.period=15s - -# Number of parallel threads used to refresh materialized view data. -# Configuration options: -# - Default: Automatically calculated based on CPU core count -# - 0: Runs as a single instance on the shared worker pool -# - N: Uses N dedicated threads for parallel refreshing -#mat.view.refresh.worker.count= - -# Materialized View refresh worker pool configuration -#mat.view.refresh.worker.affinity= -#mat.view.refresh.worker.yield.threshold=1000 -#mat.view.refresh.worker.nap.threshold=7000 -#mat.view.refresh.worker.sleep.threshold=10000 -#mat.view.refresh.worker.haltOnError=false - -# Export worker pool configuration -#export.worker.affinity= -#export.worker.yield.threshold=1000 -#export.worker.nap.threshold=7000 -#export.worker.sleep.threshold=10000 -#export.worker.haltOnError=false - -################ WAL settings ################## - -# Enable support of creating and writing to WAL tables -#cairo.wal.supported=true - -# If set to true WAL becomes the default mode for newly created tables. Impacts table created from ILP and SQL if WAL / BYPASS WAL not specified -#cairo.wal.enabled.default=true - -# Parallel threads to apply WAL data to the table storage. By default it is equal to the CPU core count. -# When set to 0 WAL apply job will run as a single instance on shared worker pool. -#wal.apply.worker.count= - -# WAL apply pool configuration -#wal.apply.worker.affinity= -#wal.apply.worker.yield.threshold=1000 -#wal.apply.worker.nap.threshold=7000 -#wal.apply.worker.sleep.threshold=10000 -#wal.apply.worker.haltOnError=false - -# Period in ms of how often WAL applied files are cleaned up from the disk -#cairo.wal.purge.interval=30s - -# Row count of how many rows are written to the same WAL segment before starting a new segment. -# Triggers in conjunction with `cairo.wal.segment.rollover.size` (whichever is first). -#cairo.wal.segment.rollover.row.count=200000 - -# Byte size of number of rows written to the same WAL segment before starting a new segment. -# Triggers in conjunction with `cairo.wal.segment.rollover.row.count` (whichever is first). -# By default this is 0 (disabled) unless `replication.role=primary` is set, then it is defaulted to 2MiB. -#cairo.wal.segment.rollover.size=0 - -# mmap sliding page size that WalWriter uses to append data for each column -#cairo.wal.writer.data.append.page.size=1M - -# mmap sliding page size that WalWriter uses to append events for each column -# this page size has performance impact on large number of small transactions, larger -# page will cope better. However, if the workload is that of small number of large transaction, -# this page size can be reduced. The optimal value should be established via ingestion benchmark. -#cairo.wal.writer.event.append.page.size=128k - -# Multiplier to cairo.max.uncommitted.rows to calculate the limit of rows that can kept invisible when writing -# to WAL table under heavy load, when multiple transactions are to be applied. -# It is used to reduce the number Out Of Order commits when Out Of Order commits are unavoidable by squashing multiple commits together. -# Setting it very low can increase Out Of Order commit frequency and decrease the throughput. -# No rows are kept in WAL lag when last committed transaction is processed. -#cairo.wal.squash.uncommitted.rows.multiplier=20.0 - -# Maximum size of data can be kept in WAL lag in bytes. -# Default is 75M and it means that once the limit is reached, 50M will be committed and 25M will be kept in the lag. -# It is used to reduce the number Out Of Order commits when Out Of Order commits are unavoidable by squashing multiple commits together. -# Setting it very low can increase Out Of Order commit frequency and decrease the throughput. -# No rows are kept in WAL lag when last committed transaction is processed. -#cairo.wal.max.lag.size=75M - -# Maximum number of transactions to keep in O3 lag for WAL tables. Once the number is reached, full commit occurs. -# By default it is -1 and the limit does not apply, instead the size limit of cairo.wal.max.lag.size is used. -# No rows are kept in WAL lag when last committed transaction is processed. -#cairo.wal.max.lag.txn.count=-1 - -# When WAL apply job processes transactions this is the minimum number of transaction -# to look ahead and read metadata of before applying any of them. -#cairo.wal.apply.look.ahead.txn.count=20 - -# Part of WAL apply job fair factor. The amount of time job spends on single table -# before moving to the next one. -#cairo.wal.apply.table.time.quota=1s - -# number of segments in the WalWriter pool; each segment holds up to 16 writers -#cairo.wal.writer.pool.max.segments=10 - -# number of segments in the ViewWalWriter pool; each segment holds up to 16 writers -#cairo.view.wal.writer.pool.max.segments=4 - -# ViewWalWriter pool releases inactive writers after this TTL. In milliseconds. -#cairo.view.wal.inactive.writer.ttl=60000 - -# Maximum number of Segments to cache File descriptors for when applying from WAL to table storage. -# WAL segment files are cached to avoid opening and closing them on processing each commit. -# Ideally should be in line with average number of simultaneous connections writing to the tables. -#cairo.wal.max.segment.file.descriptors.cache=30 - -# When disabled, SQL executed by WAL apply job always runs single-threaded. -#cairo.wal.apply.parallel.sql.enabled=true - -################ Telemetry settings ################## - -# Telemetry events are used to identify components of QuestDB that are being -# used. They never identify user nor data stored in QuestDB. All events can be -# viewed via `select * from telemetry`. Switching telemetry off will stop all -# events from being collected. After telemetry is switched off, telemetry -# tables can be dropped. - -telemetry.enabled=false -#telemetry.queue.capacity=512 -#telemetry.hide.tables=true -#telemetry.table.ttl.weeks=4 -#telemetry.event.throttle.interval=1m - -################ Metrics settings ################## - -#metrics.enabled=false - -############# Query Tracing settings ############### - -#query.tracing.enabled=false - -################ Logging settings ################## - -# enable or disable 'exe' log messages written when query execution begins -#log.sql.query.progress.exe=true - - -################ Enterprise configuration options ################## -### Please visit https://questdb.io/enterprise/ for more information -#################################################################### - - -#acl.enabled=false -#acl.entity.name.max.length=255 -#acl.admin.user.enabled=true -#acl.admin.user=admin -#acl.admin.password=quest - -## 10 seconds -#acl.rest.token.refresh.threshold=10 -#acl.sql.permission.model.pool.capacity=32 -#acl.password.hash.iteration.count=100000 - -#acl.basic.auth.realm.enabled=false - -#cold.storage.enabled=false -#cold.storage.object.store= - -#tls.enabled=false -#tls.cert.path= -#tls.private.key.path= - -## Defaults to the global TLS settings -## including enable/disable flag and paths -## Use these configuration values to separate -## min-http server TLS configuration from global settings -#http.min.tls.enabled= -#http.min.tls.cert.path= -#http.min.tls.private.key.path= - -#http.tls.enabled=false -#http.tls.cert.path= -#http.tls.private.key.path= - -#line.tcp.tls.enabled= -#line.tcp.tls.cert.path= -#line.tcp.tls.private.key.path= - -#pg.tls.enabled= -#pg.tls.cert.path= -#pg.tls.private.key.path= - -## The number of threads dedicated for async IO operations (e.g. network activity) in native code. -#native.async.io.threads= - -## The number of threads dedicated for blocking IO operations (e.g. file access) in native code. -#native.max.blocking.threads= - -### Replication (QuestDB Enterprise Only) - -# Possible roles are "primary", "replica", or "none" -# primary - read/write node -# replica - read-only node, consuming data from PRIMARY -# none - replication is disabled -#replication.role=none - -## Object-store specific string. -## AWS S3 example: -## s3::bucket=${BUCKET_NAME};root=${DB_INSTANCE_NAME};region=${AWS_REGION};access_key_id=${AWS_ACCESS_KEY};secret_access_key=${AWS_SECRET_ACCESS_KEY}; -## Azure Blob example: -## azblob::endpoint=https://${STORE_ACCOUNT}.blob.core.windows.net;container={BLOB_CONTAINER};root=${DB_INSTANCE_NAME};account_name=${STORE_ACCOUNT};account_key=${STORE_KEY}; -## Filesystem: -## fs::root=/nfs/path/to/dir/final;atomic_write_dir=/nfs/path/to/dir/scratch; -#replication.object.store= - -## Limits the number of concurrent requests to the object store. -## Defaults to 128 -#replication.requests.max.concurrent=128 - -## Maximum number of times to retry a failed object store request before -## logging an error and reattempting later after a delay. -#replication.requests.retry.attempts=3 - -## Delay between the retry attempts (milliseconds) -#replication.requests.retry.interval=200 - -## The time window grouping multiple transactions into a replication batch (milliseconds). -## Smaller time windows use more network traffic. -## Larger time windows increase the replication latency. -## Works in conjunction with `replication.primary.throttle.non.data`. -#replication.primary.throttle.window.duration=10000 - -## Set to `false` to allow immediate replication of non-data transactions -## such as table creation, rename, drop, and uploading of any closed WAL segments. -## Only set to `true` if your application is highly sensitive to network overhead. -## In most cases, tweak `cairo.wal.segment.rollover.size` and -## `replication.primary.throttle.window.duration` instead. -#replication.primary.throttle.non.data=false - -## During the upload each table's transaction log is chunked into smaller sized -## "parts". Each part defaults to 5000 transactions. Before compression, each -## transaction requires 28 bytes, and the last transaction log part is re-uploaded -## in full each time for each data upload. -## If you are heavily network constraint you may want to lower this (to say 2000), -## but this would in turn put more pressure on the object store with a larger -## amount of tiny object uploads (values lower than 1000 are never recommended). -## Note: This setting only applies to newly created tables. -#replication.primary.sequencer.part.txn.count=5000 - -## Max number of threads used to perform file compression operations before -## uploading to the object store. The default value is calculated as half the -## number of CPU cores. -#replication.primary.compression.threads= - -## Zstd compression level. Defaults to 1. Valid values are from -7 to 22. -## Where -7 is fastest and least compressed, and 22 is most CPU and memory intensive -## and most compressed. -#replication.primary.compression.level=1 - -## Whether to calculate and include a checksum with every uploaded artifact. -## By default this is is done only for services (object store implementations) -## that don't provide this themselves (typically, the filesystem). -## The other options are "never" and "always". -#replication.primary.checksum=service-dependent - -## Each time the primary instance upload a new version of the index, -## it extends the ownership time lease of the object store, ensuring no other -## primary instance may be started in its place. -## The keepalive interval determines how long to wait after a period of inactivity -## before triggering a new empty ownership-extending upload. -## This setting also determines how long the database should when migrating ownership -## to a new instance. -## The wait is calculated as `3 x $replication.primary.keepalive.interval`. -#replication.primary.keepalive.interval=10s - -## Each upload for a given table will trigger an update of the index. -## If a database has many actively replicating tables, this could trigger -## overwriting the index path on the object store many times per second. -## This can be an issue with Google Cloud Storage which has a hard-limit of -## of 1000 requests per second, per path. -## As such, we default this to a 1s grouping time window for for GCS, and no -## grouping time window for other object store providers. -#replication.primary.index.upload.throttle.interval= - -## During the uploading process WAL column files may have unused trailing data. -## The default is to skip reading/compressing/uploading this data. -#replication.primary.upload.truncated=true - -## Polling rate of a replica instance to check for new changes (milliseconds). -#replication.replica.poll.interval=1000 - -## Network buffer size (byte count) used when downloading data from the object store. -#replication.requests.buffer.size=32768 - -## How often to produce a summary of the replication progress for each table -## in the logs. The default is to do this once a minute. -## You may disable the summary by setting this parameter to 0. -#replication.summary.interval=1m - -## Enable per-table Prometheus metrics on replication progress. -## These are enabled by default. -#replication.metrics.per.table=true - -## How many times (number of HTTP Prometheus polled scrapes) should per-table -## replication metrics continue to be published for dropped tables. -## The default of 10 is designed to ensure that transaction progress -## for dropped tables is not accidentally missed. -## Consider raising if you scrape each database instance from multiple Prometheus -## instances. -#replication.metrics.dropped.table.poll.count=10 - -## Number of parallel requests to cap at when uploading or downloading data for a given -## iteration. This cap is used to perform partial progress when there is a large amount -## of outstanding uploads/downloads. -## The `fast` version is used by default for good throughput when everything is working -## well, while `slow` is used after the uploads/downloads encounter issues to allow -## progress in case of flaky networking conditions. -# replication.requests.max.batch.size.fast=64 -# replication.requests.max.batch.size.slow=2 - -## When the replication logic uploads, it times out and tries again if requests take too -## long. Since the timeout is dependent on the upload artifact size, we use a base timeout -## and a minimum throughput. The base timeout is of 10 seconds. -## The additional throughput-derived timeout is calculated from the size of the artifact -## and expressed as bytes per second. -#replication.requests.base.timeout=10s -#replication.requests.min.throughput=262144 - -### OIDC (QuestDB Enterprise Only) - -# Enables/Disables OIDC authentication. Once enabled few other -# configuration options must also be set. -#acl.oidc.enabled=false - -# Required: OIDC provider hostname -#acl.oidc.host= - -# Required: OIDC provider port number. -#acl.oidc.port=443 - -# Optional: OIDC provider host TLS setting; TLS is enabled by default. -#acl.oidc.tls.enabled=true - -# Optional: Enables QuestDB to validate TLS configuration, such as -# validity of TLS certificates. If you are working with self-signed -# certificates that you would like QuestDB to trust, disable this validations. -# Validation is strongly recommended in production environments. -#acl.oidc.tls.validation.enabled=true - -# Optional: path to keystore that contains certificate information of the -# OIDC provider. This is not required if your OIDC provider uses public CAs -#acl.oidc.tls.keystore.path= - -# Optional: keystore password, if the keystore is password protected. -#acl.oidc.tls.keystore.password= - -# Optional: OIDC provider HTTP request timeout in milliseconds -#acl.oidc.http.timeout=30000 - -# Required: OIDC provider client name. Typically OIDC provider will -# require QuestDB instance to become a "client" of the OIDC server. This -# procedure requires a "client" named entity to be created on OIDC server. -# Copy the name of the client to this configuration option -#acl.oidc.client.id= - -# Optional: OIDC authorization endpoint; the default value should work for Ping Identity Platform -#acl.oidc.authorization.endpoint=/as/authorization.oauth2 - -# Optional: OAUTH2 token endpoint; the default value should work for Ping Identity Platform -#acl.oidc.token.endpoint=/as/token.oauth2 - -# Optional: OIDC user info endpoint; the default value should work for Ping Identity Platform -#acl.oidc.userinfo.endpoint=/idp/userinfo.openid - -# Required: OIDC provider must include group array into user info response. Typically -# this is a JSON response that contains list of claim objects. Group claim is -# the name of the claim in that JSON object that contains an array of user group -# names. -#acl.oidc.groups.claim=groups - -# Optional: QuestDB instance will cache tokens for the specified TTL (millis) period before -# they are revalidated again with OIDC provider. It is a performance related setting to avoid -# otherwise stateless QuestDB instance contacting OIDC provider too frequently. -# If set to 0, the cache is disabled completely. -#acl.oidc.cache.ttl=30000 - -# Optional: QuestDB instance will cache the OIDC provider's public keys which can be used to check -# a JWT token's validity. QuestDB will reload the public keys after this expiry time to pickup -# any changes. It is a performance related setting to avoid contacting the OIDC provider too frequently. -#acl.oidc.public.keys.expiry=120000 - -# Log timestamp is generated in UTC, however, it can be written out as -# timezone of your choice. Timezone name can be either explicit UTC offset, -# such as "+08:00", "-10:00", "-07:45", "UTC+05" or timezone name "EST", "CET" etc -# https://www.joda.org/joda-time/timezones.html. Please be aware that timezone database, -# that is included with QuestDB distribution is changing from one release to the next. -# This is not something that we change, rather Java releases include timezone database -# updates. -#log.timestamp.timezone=UTC - -# Timestamp locale. You can use this to have timestamp written out using localised month -# names and day of week names. For example, for Portuguese use 'pt' locale. -#log.timestamp.locale=en - -# Format pattern used to write out log timestamps -#log.timestamp.format=yyyy-MM-ddTHH:mm:ss.SSSUUUz - -#query.within.latest.by.optimisation.enabled diff --git a/ci/questdb_start.yaml b/ci/questdb_start.yaml deleted file mode 100644 index 5799095..0000000 --- a/ci/questdb_start.yaml +++ /dev/null @@ -1,76 +0,0 @@ -parameters: - - name: configPath - type: string - -steps: - - bash: | - mkdir -p questdb-data - cp -r ${{ parameters.configPath }} questdb-data/conf - - mkdir -p logs - - # Start QuestDB in the background - java -p questdb/core/target/questdb-*-SNAPSHOT.jar -m io.questdb/io.questdb.ServerMain -d questdb-data > logs/questdb.log 2>&1 & - echo $! > questdb.pid - - # Wait for QuestDB to start - echo "Waiting for QuestDB to start..." - for i in {1..30}; do - if grep -q "server-main enjoy" logs/questdb.log; then - echo "Server started." - break - else - echo "Waiting..." - sleep 2 - fi - done - - if ! grep -q "server-main enjoy" logs/questdb.log; then - echo "QuestDB failed to start:" - cat logs/questdb.log - exit 1 - fi - displayName: "Start QuestDB server (non-Windows)" - condition: ne(variables['Agent.OS'], 'Windows_NT') - - pwsh: | - New-Item -ItemType Directory -Force -Path questdb-data | Out-Null - New-Item -ItemType Directory -Force -Path logs | Out-Null - Copy-Item -Recurse -Force -Path "${{ parameters.configPath }}" -Destination "questdb-data/conf" - - $jar = Get-ChildItem -Path "questdb/core/target" -Filter "questdb-*-SNAPSHOT.jar" | Select-Object -First 1 - if (-not $jar) { - Write-Host "QuestDB jar not found in questdb/core/target" - exit 1 - } - - $proc = Start-Process -FilePath "java" -ArgumentList @( - "-p", $jar.FullName, - "-m", "io.questdb/io.questdb.ServerMain", - "-d", "questdb-data" - ) -RedirectStandardOutput "logs/questdb.log" -RedirectStandardError "logs/questdb.err" -PassThru - - $proc.Id | Out-File -FilePath questdb.pid -Encoding ascii - - Write-Host "Waiting for QuestDB to start..." - $started = $false - for ($i = 0; $i -lt 30; $i++) { - if (Test-Path "logs/questdb.log") { - if (Select-String -Path "logs/questdb.log" -Pattern "server-main enjoy" -Quiet) { - Write-Host "Server started." - $started = $true - break - } - } - Write-Host "Waiting..." - Start-Sleep -Seconds 2 - } - - if (-not $started) { - Write-Host "QuestDB failed to start:" - if (Test-Path "logs/questdb.log") { - Get-Content "logs/questdb.log" - } - exit 1 - } - displayName: "Start QuestDB server (Windows)" - condition: eq(variables['Agent.OS'], 'Windows_NT') diff --git a/ci/questdb_stop.yaml b/ci/questdb_stop.yaml deleted file mode 100644 index 3b6c0d9..0000000 --- a/ci/questdb_stop.yaml +++ /dev/null @@ -1,39 +0,0 @@ -steps: - - bash: | - echo "Stopping QuestDB server..." - kill -TERM "$(cat questdb.pid)" || true - rm -f questdb.pid - rm -rf questdb-data - displayName: "Stop QuestDB server (non-Windows)" - condition: ne(variables['Agent.OS'], 'Windows_NT') - - pwsh: | - Write-Host "Stopping QuestDB server..." - if (Test-Path "questdb.pid") { - $questdbPid = Get-Content "questdb.pid" | Select-Object -First 1 - if ($questdbPid) { - Stop-Process -Id $questdbPid -ErrorAction SilentlyContinue - try { - Wait-Process -Id $questdbPid -Timeout 30 -ErrorAction Stop - } catch { - Stop-Process -Id $questdbPid -Force -ErrorAction SilentlyContinue - } - } - Remove-Item -Force "questdb.pid" -ErrorAction SilentlyContinue - } - if (Test-Path "questdb-data") { - $deleted = $false - for ($i = 0; $i -lt 10; $i++) { - try { - Remove-Item -Recurse -Force "questdb-data" -ErrorAction Stop - $deleted = $true - break - } catch { - Start-Sleep -Seconds 1 - } - } - if (-not $deleted) { - Remove-Item -Recurse -Force "questdb-data" - } - } - displayName: "Stop QuestDB server (Windows)" - condition: eq(variables['Agent.OS'], 'Windows_NT') diff --git a/ci/run_client_tests.yaml b/ci/run_client_tests.yaml index e8e3216..11253cd 100644 --- a/ci/run_client_tests.yaml +++ b/ci/run_client_tests.yaml @@ -1,11 +1,4 @@ -parameters: - - name: configPath - type: string - steps: - - template: questdb_start.yaml - parameters: - configPath: ${{ parameters.configPath }} - task: Maven@3 displayName: "Run Client Tests" inputs: @@ -25,4 +18,3 @@ steps: inputs: pathToPublish: $(ARCHIVED_LOGS) artifactName: MavenFailedTestsLogs - - template: questdb_stop.yaml diff --git a/ci/run_oss_tests.yaml b/ci/run_oss_tests.yaml index 91cde3a..7e8feed 100644 --- a/ci/run_oss_tests.yaml +++ b/ci/run_oss_tests.yaml @@ -1,7 +1,7 @@ # Run the tests from OSS using the current client version steps: - task: Maven@3 - displayName: "Run OSS line tests" + displayName: "Run server line tests" inputs: mavenPOMFile: "questdb/pom.xml" jdkVersionOption: "default" diff --git a/ci/run_tests_pipeline.yaml b/ci/run_tests_pipeline.yaml index cc1d618..f1c8a3f 100644 --- a/ci/run_tests_pipeline.yaml +++ b/ci/run_tests_pipeline.yaml @@ -7,8 +7,6 @@ pr: variables: ARCHIVED_LOGS: "$(Build.ArtifactStagingDirectory)/questdb-$(Build.SourceBranchName)-$(Build.SourceVersion)-$(System.StageAttempt)-$(Agent.OS).zip" - QUESTDB_RUNNING: true - QUESTDB_ILP_TCP_AUTH_ENABLE: false stages: - stage: Validate @@ -30,9 +28,9 @@ stages: displayName: "Checkout PR source branch" condition: eq(variables['Build.Reason'], 'PullRequest') - task: JavaToolInstaller@0 - displayName: "Install Java 11" + displayName: "Install Java 17" inputs: - versionSpec: "11" + versionSpec: "17" jdkArchitectureOption: "x64" jdkSourceOption: "PreInstalled" - bash: mvn -f core/pom.xml javadoc:javadoc -Pjavadoc --batch-mode @@ -72,7 +70,18 @@ stages: lfs: false submodules: false - template: setup.yaml - - script: git clone --depth 1 https://github.com/questdb/questdb.git ./questdb + - bash: echo "##vso[task.setvariable variable=HOME]$USERPROFILE" + displayName: "Set HOME on Windows" + condition: eq(variables['Agent.OS'], 'Windows_NT') + - task: Cache@2 + inputs: + key: 'maven | "$(Agent.OS)" | **/pom.xml' + restoreKeys: | + maven | "$(Agent.OS)" + path: $(HOME)/.m2/repository + displayName: "Cache Maven repository" + # TODO: remove branch once jh_experiment_new_ilp is merged + - script: git clone --depth 1 -b jh_experiment_new_ilp https://github.com/questdb/questdb.git ./questdb displayName: git clone questdb - task: Maven@3 displayName: "Update client version" @@ -88,27 +97,11 @@ stages: jdkVersionOption: "default" goals: "install" options: "-DskipTests --batch-mode" - - task: Maven@3 - displayName: "Update QuestDB client version: core" - inputs: - mavenPOMFile: "questdb/core/pom.xml" - jdkVersionOption: "default" - goals: "versions:use-dep-version" - options: "-Dincludes=org.questdb:client -DdepVersion=9.9.9 -DforceVersion=true --batch-mode" - - task: Maven@3 - displayName: "Update QuestDB client version: benchmarks" - inputs: - mavenPOMFile: "questdb/benchmarks/pom.xml" - jdkVersionOption: "default" - goals: "versions:use-dep-version" - options: "-Dincludes=org.questdb:client -DdepVersion=9.9.9 -DforceVersion=true --batch-mode" - - task: Maven@3 - displayName: "Update QuestDB client version: utils" - inputs: - mavenPOMFile: "questdb/utils/pom.xml" - jdkVersionOption: "default" - goals: "versions:use-dep-version" - options: "-Dincludes=org.questdb:client -DdepVersion=9.9.9 -DforceVersion=true --batch-mode" + - bash: | + sed -i.bak 's|.*|9.9.9|' \ + questdb/core/pom.xml questdb/benchmarks/pom.xml questdb/utils/pom.xml + rm -f questdb/core/pom.xml.bak questdb/benchmarks/pom.xml.bak questdb/utils/pom.xml.bak + displayName: "Update QuestDB client version to 9.9.9" - task: Maven@3 displayName: "Compile QuestDB" inputs: @@ -116,12 +109,4 @@ stages: jdkVersionOption: "default" options: "-DskipTests -Pbuild-web-console --batch-mode" - template: run_client_tests.yaml - parameters: - configPath: "ci/confs/default" - - bash: | - echo "##vso[task.setvariable variable=QUESTDB_ILP_TCP_AUTH_ENABLE]true" - displayName: "Enable ILP TCP Auth" - - template: run_client_tests.yaml - parameters: - configPath: "ci/confs/authenticated" - template: run_oss_tests.yaml diff --git a/core/pom.xml b/core/pom.xml index 1ccaa98..3299370 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -397,7 +397,7 @@ ${javac.target} - (11,) + [11,) diff --git a/core/src/main/c/share/net.c b/core/src/main/c/share/net.c index 644d7fa..eb5665e 100644 --- a/core/src/main/c/share/net.c +++ b/core/src/main/c/share/net.c @@ -31,6 +31,7 @@ #include #include #include +#include #include #include "net.h" #include @@ -356,4 +357,36 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_sendTo (JNIEnv *e, jclass cl, jint fd, jlong ptr, jint len, jlong sockaddr) { return (jint) sendto((int) fd, (const void *) ptr, (size_t) len, 0, (const struct sockaddr *) sockaddr, sizeof(struct sockaddr_in)); -} \ No newline at end of file +} + +JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_sendToScatter + (JNIEnv *e, jclass cl, jint fd, jlong segmentsPtr, jint segmentCount, jlong sockaddr) { + if (segmentCount <= 0) { + return 0; + } + + struct iovec *iov = calloc((size_t) segmentCount, sizeof(struct iovec)); + if (iov == NULL) { + errno = ENOMEM; + return -1; + } + + const char *segment = (const char *) segmentsPtr; + for (int i = 0; i < segmentCount; i++) { + iov[i].iov_base = (void *) (uintptr_t) (*(const jlong *) segment); + iov[i].iov_len = (size_t) (*(const jlong *) (segment + 8)); + segment += 16; + } + + struct msghdr msg; + memset(&msg, 0, sizeof(msg)); + msg.msg_name = (void *) sockaddr; + msg.msg_namelen = sizeof(struct sockaddr_in); + msg.msg_iov = iov; + msg.msg_iovlen = (size_t) segmentCount; + + ssize_t sent; + RESTARTABLE(sendmsg((int) fd, &msg, 0), sent); + free(iov); + return sent < 0 ? -1 : (jint) sent; +} diff --git a/core/src/main/c/share/net.h b/core/src/main/c/share/net.h index c3f1016..13adafc 100644 --- a/core/src/main/c/share/net.h +++ b/core/src/main/c/share/net.h @@ -182,6 +182,14 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_setMulticastTtl JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_sendTo (JNIEnv *, jclass, jint, jlong, jint, jlong); +/* + * Class: io_questdb_client_network_Net + * Method: sendToScatter + * Signature: (IJIJ)I + */ +JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_sendToScatter + (JNIEnv *, jclass, jint, jlong, jint, jlong); + #ifdef __cplusplus } #endif diff --git a/core/src/main/c/windows/net.c b/core/src/main/c/windows/net.c index eb25fef..1056ab4 100644 --- a/core/src/main/c/windows/net.c +++ b/core/src/main/c/windows/net.c @@ -25,6 +25,7 @@ #include #include #include +#include #include "../share/net.h" #include "errno.h" @@ -287,4 +288,45 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_sendTo SaveLastError(); } return result; -} \ No newline at end of file +} + +JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_sendToScatter + (JNIEnv *e, jclass cl, jint fd, jlong segmentsPtr, jint segmentCount, jlong sockaddr) { + if (segmentCount <= 0) { + return 0; + } + + WSABUF *buffers = calloc((size_t) segmentCount, sizeof(WSABUF)); + if (buffers == NULL) { + WSASetLastError(WSA_NOT_ENOUGH_MEMORY); + SaveLastError(); + return -1; + } + + const char *segment = (const char *) segmentsPtr; + for (int i = 0; i < segmentCount; i++) { + buffers[i].buf = (CHAR *) (uintptr_t) (*(const jlong *) segment); + buffers[i].len = (ULONG) (*(const jlong *) (segment + 8)); + segment += 16; + } + + DWORD bytesSent = 0; + int result = WSASendTo( + (SOCKET) fd, + buffers, + (DWORD) segmentCount, + &bytesSent, + 0, + (const struct sockaddr *) sockaddr, + sizeof(struct sockaddr_in), + NULL, + NULL + ); + free(buffers); + + if (result == SOCKET_ERROR) { + SaveLastError(); + return -1; + } + return (jint) bytesSent; +} diff --git a/core/src/main/java/io/questdb/client/BuildInformationHolder.java b/core/src/main/java/io/questdb/client/BuildInformationHolder.java index fbf34d8..763d62d 100644 --- a/core/src/main/java/io/questdb/client/BuildInformationHolder.java +++ b/core/src/main/java/io/questdb/client/BuildInformationHolder.java @@ -41,7 +41,7 @@ public BuildInformationHolder(Class clazz) { String swVersion; try { final Attributes manifestAttributes = getManifestAttributes(clazz); - swVersion = getAttr(manifestAttributes, "QuestDB-Client-Version", "[DEVELOPMENT]"); + swVersion = getAttr(manifestAttributes, "[DEVELOPMENT]"); } catch (IOException e) { swVersion = UNKNOWN; } @@ -57,8 +57,8 @@ public String getSwVersion() { return swVersion; } - private static String getAttr(final Attributes manifestAttributes, String attributeName, String defaultValue) { - final String value = manifestAttributes.getValue(attributeName); + private static String getAttr(final Attributes manifestAttributes, String defaultValue) { + final String value = manifestAttributes.getValue("QuestDB-Client-Version"); return value != null ? value : defaultValue; } diff --git a/core/src/main/java/io/questdb/client/HttpClientConfiguration.java b/core/src/main/java/io/questdb/client/HttpClientConfiguration.java index 7ed2741..0fe6c6e 100644 --- a/core/src/main/java/io/questdb/client/HttpClientConfiguration.java +++ b/core/src/main/java/io/questdb/client/HttpClientConfiguration.java @@ -54,6 +54,10 @@ default int getMaximumRequestBufferSize() { return Integer.MAX_VALUE; } + default int getMaximumResponseBufferSize() { + return Integer.MAX_VALUE; + } + default NetworkFacade getNetworkFacade() { return NetworkFacadeImpl.INSTANCE; } diff --git a/core/src/main/java/io/questdb/client/ParanoiaState.java b/core/src/main/java/io/questdb/client/ParanoiaState.java deleted file mode 100644 index 90e15e1..0000000 --- a/core/src/main/java/io/questdb/client/ParanoiaState.java +++ /dev/null @@ -1,84 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client; - -// Constants that enable various diagnostics to catch leaks, double closes, etc. -public class ParanoiaState { - /** - * 
-     * BASIC -> validates UTF-8 in log records (throws a LogError if invalid),
-     *          throws a LogError on abandoned log records (missing .$() at the end of log statement),
-     *          detects closed stdout in LogConsoleWriter.
-     *          This introduces a low overhead to logging.
-     * AGGRESSIVE -> BASIC + holds recent history of log lines to help diagnose closed stdout,
-     *               holds the stack trace of abandoned log record.
-     *               This introduces a significant overhead to logging.
-     *
-     * When running inside JUnit/Surefire, BASIC log paranoia mode gets activated automatically.
-     * You can manually edit the code in the static { } block below to activate AGGRESSIVE instead.
-     *
-     * Logs may go silent when Maven Surefire plugin closes stdout due to broken text encoding.
-     * In BASIC mode, the log writer will detect this and print errors through System.out, which
-     * under Surefire uses an alternate channel and not stdout.
-     * In AGGRESSIVE mode, it will additionally remember the most recent log lines and print them.
-     * This will help you find the offending log line with broken encoding.
-     *
-     * The logging framework detects a common coding error where you forget to end a log statement
-     * with .$(), causing the statement not to be logged. This problem can only be detected after
-     * the fact, when you start a new log record and the previous one wasn't completed.
-     *
-     * With Log Paranoia off (LOG_PARANOIA_MODE_NONE), we only detect this problem and print an
-     * error message.
-     * In BASIC mode, we throw a LogError without a stack trace.
-     * In AGGRESSIVE mode, we capture the stack trace at every start of a log statement, so when
-     * we throw the LogError, it points to the code that created and then abandoned the log record.
-     *
- */ - public static final int LOG_PARANOIA_MODE; - public static final int LOG_PARANOIA_MODE_AGGRESSIVE = 2; - public static final int LOG_PARANOIA_MODE_BASIC = 1; - public static final int LOG_PARANOIA_MODE_NONE = 0; - // Set to true to enable Thread Local path instances created/closed stack trace logs. - public static final boolean THREAD_LOCAL_PATH_PARANOIA_MODE = false; - // Set to true to enable stricter boundary checks on Vm memories implementations. - public static final boolean VM_PARANOIA_MODE = false; - // Set to true to enable stricter File Descriptor double close checks, trace closed usages. - public static boolean FD_PARANOIA_MODE = false; - - public static boolean isInsideJUnitTest() { - StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace(); - for (StackTraceElement element : stackTrace) { - String className = element.getClassName(); - if (className.startsWith("org.apache.maven.surefire") || className.startsWith("org.junit.")) { - return true; - } - } - return false; - } - - static { - LOG_PARANOIA_MODE = isInsideJUnitTest() ? LOG_PARANOIA_MODE_BASIC : LOG_PARANOIA_MODE_NONE; - } -} \ No newline at end of file diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index aea44e7..9811e24 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -34,6 +34,8 @@ import io.questdb.client.cutlass.line.http.AbstractLineHttpSender; import io.questdb.client.cutlass.line.tcp.DelegatingTlsChannel; import io.questdb.client.cutlass.line.tcp.PlainTcpLineChannel; +import io.questdb.client.cutlass.qwp.client.QwpUdpSender; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.impl.ConfStringParser; import io.questdb.client.network.NetworkFacade; import io.questdb.client.network.NetworkFacadeImpl; @@ -51,9 +53,13 @@ import javax.security.auth.DestroyFailedException; import java.io.Closeable; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.nio.charset.StandardCharsets; import java.security.PrivateKey; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.Base64; import java.util.concurrent.TimeUnit; /** @@ -95,7 +101,7 @@ * 2. Call {@link #reset()} to clear the internal buffers and start building a new row *
* Note: If the underlying error is permanent, retrying {@link #flush()} will fail again. - * Use {@link #reset()} to discard the problematic data and continue with new data. See {@link LineSenderException#isRetryable()} + * Use {@link #reset()} to discard the problematic data and continue with new data. * */ public interface Sender extends Closeable, ArraySender { @@ -108,7 +114,7 @@ public interface Sender extends Closeable, ArraySender { /** * Create a Sender builder instance from a configuration string. *
- * This allows to use the configuration string as a template for creating a Sender builder instance and then + * This allows using the configuration string as a template for creating a Sender builder instance and then * tune options which are not available in the configuration string. Configurations options specified in the * configuration string cannot be overridden via the builder methods. *

@@ -144,7 +150,24 @@ static LineSenderBuilder builder(CharSequence configurationString) { * @return Builder object to create a new Sender instance. */ static LineSenderBuilder builder(Transport transport) { - return new LineSenderBuilder(transport == Transport.HTTP ? LineSenderBuilder.PROTOCOL_HTTP : LineSenderBuilder.PROTOCOL_TCP); + int protocol; + switch (transport) { + case HTTP: + protocol = LineSenderBuilder.PROTOCOL_HTTP; + break; + case TCP: + protocol = LineSenderBuilder.PROTOCOL_TCP; + break; + case UDP: + protocol = LineSenderBuilder.PROTOCOL_UDP; + break; + case WEBSOCKET: + protocol = LineSenderBuilder.PROTOCOL_WEBSOCKET; + break; + default: + throw new IllegalArgumentException("unknown transport: " + transport); + } + return new LineSenderBuilder(protocol); } /** @@ -461,7 +484,23 @@ enum Transport { * and for use-cases where HTTP transport is not suitable, when communicating with a QuestDB server over a high-latency * network */ - TCP + TCP, + + /** + * Fire-and-forget binary ingestion over UDP. + *

+ * UDP transport sends datagrams without waiting for acknowledgement. It is suitable for + * high-throughput scenarios where occasional message loss is acceptable. + */ + UDP, + + /** + * Use WebSocket transport to communicate with a QuestDB server. + *

+ * WebSocket transport uses the QWP v1 binary protocol for efficient data ingestion. + * It supports both synchronous and asynchronous modes with flow control. + */ + WEBSOCKET } /** @@ -508,12 +547,19 @@ final class LineSenderBuilder { private static final int DEFAULT_BUFFER_CAPACITY = 64 * 1024; private static final int DEFAULT_HTTP_PORT = 9000; private static final int DEFAULT_HTTP_TIMEOUT = 30_000; + private static final int DEFAULT_IN_FLIGHT_WINDOW_SIZE = 128; private static final int DEFAULT_MAXIMUM_BUFFER_CAPACITY = 100 * 1024 * 1024; private static final int DEFAULT_MAX_BACKOFF_MILLIS = 1_000; + private static final int DEFAULT_MAX_DATAGRAM_SIZE = 1400; private static final int DEFAULT_MAX_NAME_LEN = 127; private static final long DEFAULT_MAX_RETRY_NANOS = TimeUnit.SECONDS.toNanos(10); // keep sync with the contract of the configuration method private static final long DEFAULT_MIN_REQUEST_THROUGHPUT = 100 * 1024; // 100KB/s, keep in sync with the contract of the configuration method private static final int DEFAULT_TCP_PORT = 9009; + private static final int DEFAULT_UDP_PORT = 9007; + private static final int DEFAULT_WEBSOCKET_PORT = 9000; + private static final int DEFAULT_WS_AUTO_FLUSH_BYTES = 128 * 1024; // 128KB + private static final long DEFAULT_WS_AUTO_FLUSH_INTERVAL_NANOS = 100_000_000L; // 100ms + private static final int DEFAULT_WS_AUTO_FLUSH_ROWS = 1_000; private static final int MIN_BUFFER_SIZE = AuthUtils.CHALLENGE_LEN + 1; // challenge size + 1; // The PARAMETER_NOT_SET_EXPLICITLY constant is used to detect if a parameter was set explicitly in configuration parameters // where it matters. This is needed to detect invalid combinations of parameters. Why? @@ -522,8 +568,11 @@ final class LineSenderBuilder { private static final int PARAMETER_NOT_SET_EXPLICITLY = -1; private static final int PROTOCOL_HTTP = 1; private static final int PROTOCOL_TCP = 0; + private static final int PROTOCOL_UDP = 3; + private static final int PROTOCOL_WEBSOCKET = 2; private final ObjList hosts = new ObjList<>(); private final IntList ports = new IntList(); + private int autoFlushBytes = PARAMETER_NOT_SET_EXPLICITLY; private int autoFlushIntervalMillis = PARAMETER_NOT_SET_EXPLICITLY; private int autoFlushRows = PARAMETER_NOT_SET_EXPLICITLY; private int bufferCapacity = PARAMETER_NOT_SET_EXPLICITLY; @@ -531,8 +580,10 @@ final class LineSenderBuilder { private String httpSettingsPath; private int httpTimeout = PARAMETER_NOT_SET_EXPLICITLY; private String httpToken; + private int inFlightWindowSize = PARAMETER_NOT_SET_EXPLICITLY; private String keyId; private int maxBackoffMillis = PARAMETER_NOT_SET_EXPLICITLY; + private int maxDatagramSize = PARAMETER_NOT_SET_EXPLICITLY; private int maxNameLength = PARAMETER_NOT_SET_EXPLICITLY; private int maximumBufferCapacity = PARAMETER_NOT_SET_EXPLICITLY; private final HttpClientConfiguration httpClientConfiguration = new DefaultHttpClientConfiguration() { @@ -557,6 +608,7 @@ public int getTimeout() { } }; private long minRequestThroughput = PARAMETER_NOT_SET_EXPLICITLY; + private int multicastTtl = PARAMETER_NOT_SET_EXPLICITLY; private String password; private PrivateKey privateKey; private int protocol = PARAMETER_NOT_SET_EXPLICITLY; @@ -658,6 +710,45 @@ public AdvancedTlsSettings advancedTls() { return new AdvancedTlsSettings(); } + /** + * @deprecated Async mode is now derived from {@link #inFlightWindowSize(int)}. + * Window size 1 implies synchronous mode, greater than 1 implies asynchronous mode. + * The default window size is 128 (asynchronous). Call {@code inFlightWindowSize(1)} + * for synchronous behavior. + *
+ * This method is a no-op and will be removed in a future release. + * + * @param enabled ignored + * @return this instance for method chaining + */ + @Deprecated + public LineSenderBuilder asyncMode(boolean enabled) { + return this; + } + + /** + * Set the maximum number of bytes per batch before auto-flushing. + *
+ * This is only used when communicating over WebSocket transport. + *
+ * Default value is 128KB. + * + * @param bytes maximum bytes per batch + * @return this instance for method chaining + */ + public LineSenderBuilder autoFlushBytes(int bytes) { + if (this.autoFlushBytes != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("auto flush bytes was already configured") + .put("[bytes=").put(this.autoFlushBytes).put("]"); + } + if (bytes < 0) { + throw new LineSenderException("auto flush bytes cannot be negative") + .put("[bytes=").put(bytes).put("]"); + } + this.autoFlushBytes = bytes; + return this; + } + /** * Set the interval in milliseconds at which the Sender automatically flushes its buffer. *
@@ -791,6 +882,43 @@ public Sender build() { username, password, maxNameLength, actualMaxRetriesNanos, maxBackoffMillis, actualMinRequestThroughput, actualAutoFlushIntervalMillis, protocolVersion); } + if (protocol == PROTOCOL_WEBSOCKET) { + if (hosts.size() != 1 || ports.size() != 1) { + throw new LineSenderException("only a single address (host:port) is supported for WebSocket transport"); + } + + int actualAutoFlushRows = autoFlushRows == PARAMETER_NOT_SET_EXPLICITLY ? DEFAULT_WS_AUTO_FLUSH_ROWS : autoFlushRows; + int actualAutoFlushBytes = autoFlushBytes == PARAMETER_NOT_SET_EXPLICITLY ? DEFAULT_WS_AUTO_FLUSH_BYTES : autoFlushBytes; + long actualAutoFlushIntervalNanos = autoFlushIntervalMillis == PARAMETER_NOT_SET_EXPLICITLY + ? DEFAULT_WS_AUTO_FLUSH_INTERVAL_NANOS + : TimeUnit.MILLISECONDS.toNanos(autoFlushIntervalMillis); + int actualInFlightWindowSize = inFlightWindowSize == PARAMETER_NOT_SET_EXPLICITLY ? DEFAULT_IN_FLIGHT_WINDOW_SIZE : inFlightWindowSize; + + String wsAuthHeader = buildWebSocketAuthHeader(); + + return QwpWebSocketSender.connect( + hosts.getQuick(0), + ports.getQuick(0), + tlsEnabled, + actualAutoFlushRows, + actualAutoFlushBytes, + actualAutoFlushIntervalNanos, + actualInFlightWindowSize, + wsAuthHeader + ); + } + + if (protocol == PROTOCOL_UDP) { + if (hosts.size() != 1 || ports.size() != 1) { + throw new LineSenderException("only a single address (host:port) is supported for UDP transport"); + } + int sendToAddr = resolveIPv4(hosts.getQuick(0)); + int actualMaxDatagramSize = maxDatagramSize == PARAMETER_NOT_SET_EXPLICITLY + ? DEFAULT_MAX_DATAGRAM_SIZE : maxDatagramSize; + int actualTtl = multicastTtl == PARAMETER_NOT_SET_EXPLICITLY ? 0 : multicastTtl; + return new QwpUdpSender(nf, 0, sendToAddr, ports.getQuick(0), actualTtl, actualMaxDatagramSize); + } + assert protocol == PROTOCOL_TCP; if (hosts.size() != 1 || ports.size() != 1) { @@ -1048,6 +1176,32 @@ public LineSenderBuilder httpUsernamePassword(String username, String password) return this; } + /** + * Set the maximum number of batches that can be in-flight awaiting server acknowledgment. + *
+ * This is only used when communicating over WebSocket transport. + *
+ * A value of 1 means synchronous mode: each batch waits for an ACK before sending the next one. + * A value greater than 1 enables asynchronous mode with pipelined sends and a background I/O thread. + *
+ * Default value is 128 (asynchronous). + * + * @param size maximum number of in-flight batches + * @return this instance for method chaining + */ + public LineSenderBuilder inFlightWindowSize(int size) { + if (this.inFlightWindowSize != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("in-flight window size was already configured") + .put("[size=").put(this.inFlightWindowSize).put("]"); + } + if (size < 1) { + throw new LineSenderException("in-flight window size must be positive") + .put("[size=").put(size).put("]"); + } + this.inFlightWindowSize = size; + return this; + } + /** * Configures the maximum backoff time between retry attempts when the Sender encounters recoverable errors. *
@@ -1114,6 +1268,32 @@ public LineSenderBuilder maxBufferCapacity(int maximumBufferCapacity) { return this; } + /** + * Set the maximum datagram size in bytes for UDP transport. Only valid for UDP transport. + *
+ * The practical limit depends on the network MTU (typically 1500 bytes for Ethernet). + *
+ * Default value: 1400 bytes + * + * @param maxDatagramSize maximum datagram size in bytes + * @return this instance for method chaining + */ + public LineSenderBuilder maxDatagramSize(int maxDatagramSize) { + if (this.maxDatagramSize != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("max datagram size was already configured ") + .put("[maxDatagramSize=").put(this.maxDatagramSize).put("]"); + } + if (maxDatagramSize < 1) { + throw new LineSenderException("max datagram size must be positive ") + .put("[maxDatagramSize=").put(maxDatagramSize).put("]"); + } + if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_UDP) { + throw new LineSenderException("max datagram size is only supported for UDP transport"); + } + this.maxDatagramSize = maxDatagramSize; + return this; + } + /** * Set the maximum length of a table or column name in bytes. * Matches the `cairo.max.file.name.length` setting in the server. @@ -1161,6 +1341,36 @@ public LineSenderBuilder minRequestThroughput(int minRequestThroughput) { return this; } + /** + * Set the multicast TTL for UDP transport. Only valid for UDP transport. + *
+ * Valid range: 0-255. + *
+ * Default value: 0 (restricted to same host). Set to 1 for local subnet. + * + * @param multicastTtl multicast TTL value + * @return this instance for method chaining + */ + public LineSenderBuilder multicastTtl(int multicastTtl) { + if (this.multicastTtl != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("multicast TTL was already configured ") + .put("[multicastTtl=").put(this.multicastTtl).put("]"); + } + if (multicastTtl < 0) { + throw new LineSenderException("multicast TTL cannot be negative ") + .put("[multicastTtl=").put(multicastTtl).put("]"); + } + if (multicastTtl > 255) { + throw new LineSenderException("multicast TTL cannot exceed 255 ") + .put("[multicastTtl=").put(multicastTtl).put("]"); + } + if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_UDP) { + throw new LineSenderException("multicast TTL is only supported for UDP transport"); + } + this.multicastTtl = multicastTtl; + return this; + } + /** * Set port where a QuestDB server is listening on. * @@ -1257,6 +1467,21 @@ private static int parseIntValue(@NotNull StringSink value, @NotNull String name } } + private static int resolveIPv4(String host) { + try { + byte[] addr = InetAddress.getByName(host).getAddress(); + if (addr.length != 4) { + throw new LineSenderException("IPv6 addresses are not supported [host=").put(host).put("]"); + } + return ((addr[0] & 0xFF) << 24) + | ((addr[1] & 0xFF) << 16) + | ((addr[2] & 0xFF) << 8) + | (addr[3] & 0xFF); + } catch (UnknownHostException e) { + throw new LineSenderException("could not resolve host [host=" + host + "]", e); + } + } + private static RuntimeException rethrow(Throwable t) { if (t instanceof LineSenderException) { throw (LineSenderException) t; @@ -1264,6 +1489,17 @@ private static RuntimeException rethrow(Throwable t) { throw new LineSenderException(t); } + private String buildWebSocketAuthHeader() { + if (username != null && password != null) { + String credentials = username + ":" + password; + return "Basic " + Base64.getEncoder().encodeToString(credentials.getBytes(StandardCharsets.UTF_8)); + } + if (httpToken != null) { + return "Bearer " + httpToken; + } + return null; + } + private void configureDefaults() { if (protocol == PARAMETER_NOT_SET_EXPLICITLY) { protocol = PROTOCOL_TCP; @@ -1275,7 +1511,15 @@ private void configureDefaults() { maximumBufferCapacity = protocol == PROTOCOL_HTTP ? DEFAULT_MAXIMUM_BUFFER_CAPACITY : bufferCapacity; } if (ports.size() == 0) { - ports.add(protocol == PROTOCOL_HTTP ? DEFAULT_HTTP_PORT : DEFAULT_TCP_PORT); + if (protocol == PROTOCOL_HTTP) { + ports.add(DEFAULT_HTTP_PORT); + } else if (protocol == PROTOCOL_UDP) { + ports.add(DEFAULT_UDP_PORT); + } else if (protocol == PROTOCOL_WEBSOCKET) { + ports.add(DEFAULT_WEBSOCKET_PORT); + } else { + ports.add(DEFAULT_TCP_PORT); + } } if (tlsValidationMode == null) { tlsValidationMode = TlsValidationMode.DEFAULT; @@ -1287,7 +1531,7 @@ private void configureDefaults() { if (maxNameLength == PARAMETER_NOT_SET_EXPLICITLY) { maxNameLength = DEFAULT_MAX_NAME_LEN; } - if (maxBackoffMillis == PARAMETER_NOT_SET_EXPLICITLY) { + if (maxBackoffMillis == PARAMETER_NOT_SET_EXPLICITLY && protocol == PROTOCOL_HTTP) { maxBackoffMillis = DEFAULT_MAX_BACKOFF_MILLIS; } } @@ -1314,9 +1558,24 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { throw new LineSenderException("invalid configuration string: ").put(sink); } if (protocol != PARAMETER_NOT_SET_EXPLICITLY) { + String protocolName; + switch (protocol) { + case PROTOCOL_HTTP: + protocolName = "http"; + break; + case PROTOCOL_UDP: + protocolName = "udp"; + break; + case PROTOCOL_WEBSOCKET: + protocolName = "websocket"; + break; + default: + protocolName = "tcp"; + break; + } throw new LineSenderException("protocol was already configured ") .put("[protocol=") - .put(protocol == PROTOCOL_HTTP ? "http" : "tcp").put("]"); + .put(protocolName).put("]"); } if (Chars.equals("http", sink)) { if (tlsEnabled) { @@ -1334,8 +1593,20 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { } else if (Chars.equals("tcps", sink)) { tcp(); tlsEnabled = true; + } else if (Chars.equals("ws", sink)) { + if (tlsEnabled) { + throw new LineSenderException("cannot use ws protocol when TLS is enabled. use wss instead"); + } + websocket(); + } else if (Chars.equals("wss", sink)) { + websocket(); + tlsEnabled = true; + } else if (Chars.equals("udp", sink)) { + udp(); + } else if (Chars.equals("udps", sink)) { + throw new LineSenderException("TLS is not supported for UDP"); } else { - throw new LineSenderException("invalid schema [schema=").put(sink).put(", supported-schemas=[http, https, tcp, tcps]]"); + throw new LineSenderException("invalid schema [schema=").put(sink).put(", supported-schemas=[http, https, tcp, tcps, ws, wss, udp]]"); } String tcpToken = null; @@ -1357,26 +1628,39 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { address(sink); if (ports.size() == hosts.size() - 1) { // not set - port(protocol == PROTOCOL_TCP ? DEFAULT_TCP_PORT : DEFAULT_HTTP_PORT); + port(protocol == PROTOCOL_HTTP ? DEFAULT_HTTP_PORT + : protocol == PROTOCOL_UDP ? DEFAULT_UDP_PORT + : protocol == PROTOCOL_WEBSOCKET ? DEFAULT_WEBSOCKET_PORT + : DEFAULT_TCP_PORT); } } else if (Chars.equals("user", sink)) { // deprecated key: user, new key: username pos = getValue(configurationString, pos, sink, "user"); + if (protocol == PROTOCOL_UDP) { + throw new LineSenderException("username is not supported for UDP transport"); + } user = sink.toString(); } else if (Chars.equals("username", sink)) { pos = getValue(configurationString, pos, sink, "username"); + if (protocol == PROTOCOL_UDP) { + throw new LineSenderException("username is not supported for UDP transport"); + } user = sink.toString(); } else if (Chars.equals("pass", sink)) { // deprecated key: pass, new key: password pos = getValue(configurationString, pos, sink, "pass"); if (protocol == PROTOCOL_TCP) { throw new LineSenderException("password is not supported for TCP protocol"); + } else if (protocol == PROTOCOL_UDP) { + throw new LineSenderException("password is not supported for UDP transport"); } password = sink.toString(); } else if (Chars.equals("password", sink)) { pos = getValue(configurationString, pos, sink, "password"); if (protocol == PROTOCOL_TCP) { throw new LineSenderException("password is not supported for TCP protocol"); + } else if (protocol == PROTOCOL_UDP) { + throw new LineSenderException("password is not supported for UDP transport"); } password = sink.toString(); } else if (Chars.equals("tls_verify", sink)) { @@ -1408,13 +1692,15 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { } } else if (Chars.equals("token", sink)) { pos = getValue(configurationString, pos, sink, "token"); - if (protocol == PROTOCOL_TCP) { + if (protocol == PROTOCOL_WEBSOCKET) { + throw new LineSenderException("token is not supported for WebSocket protocol"); + } else if (protocol == PROTOCOL_TCP) { tcpToken = sink.toString(); // will configure later, we need to know a keyId first - } else if (protocol == PROTOCOL_HTTP) { - httpToken(sink.toString()); + } else if (protocol == PROTOCOL_UDP) { + throw new LineSenderException("token is not supported for UDP transport"); } else { - throw new AssertionError(); + httpToken(sink.toString()); } } else if (Chars.equals("retry_timeout", sink)) { pos = getValue(configurationString, pos, sink, "retry_timeout"); @@ -1503,6 +1789,21 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { int protocolVersion = parseIntValue(sink, "protocol_version"); protocolVersion(protocolVersion); } + } else if (Chars.equals("in_flight_window", sink)) { + if (protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("in_flight_window is only supported for WebSocket transport"); + } + pos = getValue(configurationString, pos, sink, "in_flight_window"); + int windowSize = parseIntValue(sink, "in_flight_window"); + inFlightWindowSize(windowSize); + } else if (Chars.equals("max_datagram_size", sink)) { + pos = getValue(configurationString, pos, sink, "max_datagram_size"); + int mds = parseIntValue(sink, "max_datagram_size"); + maxDatagramSize(mds); + } else if (Chars.equals("multicast_ttl", sink)) { + pos = getValue(configurationString, pos, sink, "multicast_ttl"); + int ttl = parseIntValue(sink, "multicast_ttl"); + multicastTtl(ttl); } else { // ignore unknown keys, unless they are malformed if ((pos = ConfStringParser.value(configurationString, pos, sink)) < 0) { @@ -1520,11 +1821,11 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { } else if (trustStorePassword != null) { throw new LineSenderException("tls_roots_password was configured, but tls_roots is missing"); } - if (protocol == PROTOCOL_HTTP) { + if (protocol == PROTOCOL_HTTP || protocol == PROTOCOL_WEBSOCKET) { if (user != null) { httpUsernamePassword(user, password); } else if (password != null) { - throw new LineSenderException("HTTP password is configured, but username is missing"); + throw new LineSenderException("password is configured, but username is missing"); } } else { if (user != null) { @@ -1557,6 +1858,14 @@ private void tcp() { protocol = PROTOCOL_TCP; } + private void udp() { + if (protocol != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("protocol was already configured ") + .put("[protocol=").put(protocol).put("]"); + } + protocol = PROTOCOL_UDP; + } + private void validateParameters() { if (hosts.size() == 0) { throw new LineSenderException("questdb server address not set"); @@ -1617,12 +1926,91 @@ private void validateParameters() { if (autoFlushIntervalMillis != PARAMETER_NOT_SET_EXPLICITLY) { throw new LineSenderException("auto flush interval is not supported for TCP protocol"); } + } else if (protocol == PROTOCOL_UDP) { + if (privateKey != null) { + throw new LineSenderException("authentication is not supported for UDP transport"); + } + if (httpToken != null) { + throw new LineSenderException("HTTP token authentication is not supported for UDP transport"); + } + if (username != null || password != null) { + throw new LineSenderException("username/password authentication is not supported for UDP transport"); + } + if (tlsEnabled) { + throw new LineSenderException("TLS is not supported for UDP transport"); + } + if (retryTimeoutMillis != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("retry timeout is not supported for UDP transport"); + } + if (httpTimeout != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("HTTP timeout is not supported for UDP transport"); + } + if (minRequestThroughput != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("minimum request throughput is not supported for UDP transport"); + } + if (protocolVersion != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("protocol version is not supported for UDP transport"); + } + if (inFlightWindowSize != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("in-flight window size is not supported for UDP transport"); + } + if (httpPath != null) { + throw new LineSenderException("HTTP path is not supported for UDP transport"); + } + if (maxBackoffMillis != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("max backoff is not supported for UDP transport"); + } + if (autoFlushRows != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("auto flush rows is not supported for UDP transport"); + } + if (autoFlushIntervalMillis != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("auto flush interval is not supported for UDP transport"); + } + if (autoFlushBytes != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("auto flush bytes is not supported for UDP transport"); + } + } else if (protocol == PROTOCOL_WEBSOCKET) { + if (privateKey != null) { + throw new LineSenderException("TCP authentication is not supported for WebSocket protocol"); + } + if (httpToken != null && (username != null || password != null)) { + throw new LineSenderException("cannot use both token and username/password authentication"); + } + if (httpPath != null) { + throw new LineSenderException("HTTP path is not supported for WebSocket protocol"); + } + if (httpTimeout != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("HTTP timeout is not supported for WebSocket protocol"); + } + if (retryTimeoutMillis != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("retry timeout is not supported for WebSocket protocol"); + } + if (minRequestThroughput != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("minimum request throughput is not supported for WebSocket protocol"); + } + if (maxBackoffMillis != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("max backoff is not supported for WebSocket protocol"); + } + if (protocolVersion != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("protocol version is not supported for WebSocket protocol"); + } + if (autoFlushIntervalMillis == Integer.MAX_VALUE) { + throw new LineSenderException("disabling auto-flush is not supported for WebSocket protocol"); + } } else { throw new LineSenderException("unsupported protocol ") .put("[protocol=").put(protocol).put("]"); } } + private void websocket() { + if (protocol != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("protocol was already configured ") + .put("[protocol=").put(protocol).put("]"); + } + protocol = PROTOCOL_WEBSOCKET; + } + public class AdvancedTlsSettings { /** * Configure a custom truststore. This is only needed when using {@link #enableTls()} when your default diff --git a/core/src/main/java/io/questdb/client/cairo/ColumnType.java b/core/src/main/java/io/questdb/client/cairo/ColumnType.java index 98d4aba..74ba886 100644 --- a/core/src/main/java/io/questdb/client/cairo/ColumnType.java +++ b/core/src/main/java/io/questdb/client/cairo/ColumnType.java @@ -214,19 +214,6 @@ public static int encodeArrayType(int elemType, int nDims, boolean checkSupporte | ARRAY; } - /** - * Encodes an array type with weak dimensionality. The dimensionality is still - * encoded but marked as tentative and can be updated based on actual data. - * This is useful for PostgreSQL wire protocol where type information doesn't - * include array dimensions. - *

- * The number of dimensions of this type is undefined, so the decoded number on - * dimensions for the returned column type will be -1. - */ - public static int encodeArrayTypeWithWeakDims(short elemType, boolean checkSupportedElementTypes) { - return encodeArrayType(elemType, 1, checkSupportedElementTypes) | TYPE_FLAG_ARRAY_WEAK_DIMS; - } - /** * Generate a decimal type from a given precision and scale. * It will choose the proper subtype (DECIMAL8, DECIMAL16, etc.) from the precision, depending on the amount @@ -365,6 +352,7 @@ private static int mkGeoHashType(int bits, short baseType) { typeNameMap.put(NULL, "NULL"); arrayTypeSet.add(DOUBLE); + arrayTypeSet.add(LONG); TYPE_SIZE_POW2[UNDEFINED] = -1; TYPE_SIZE_POW2[BOOLEAN] = 0; @@ -517,4 +505,4 @@ private static int mkGeoHashType(int bits, short baseType) { } } } -} \ No newline at end of file +} diff --git a/core/src/main/java/io/questdb/client/cairo/GeoHashes.java b/core/src/main/java/io/questdb/client/cairo/GeoHashes.java deleted file mode 100644 index 4bd4028..0000000 --- a/core/src/main/java/io/questdb/client/cairo/GeoHashes.java +++ /dev/null @@ -1,107 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.cairo; - -import io.questdb.client.std.Numbers; -import io.questdb.client.std.NumericException; -import io.questdb.client.std.str.CharSink; - -public class GeoHashes { - - // geohash null value: -1 - // we use the highest bit of every storage size (byte, short, int, long) - // to indicate null value. When a null value is cast down, nullity is - // preserved, i.e. highest bit remains set: - // long nullLong = -1L; - // short nullShort = (short) nullLong; - // nullShort == nullLong; - // in addition, -1 is the first negative non geohash value. - public static final int MAX_STRING_LENGTH = 12; - public static final long NULL = -1L; - - private static final char[] base32 = { - '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', 'b', 'c', 'd', 'e', 'f', 'g', - 'h', 'j', 'k', 'm', 'n', 'p', 'q', 'r', - 's', 't', 'u', 'v', 'w', 'x', 'y', 'z' - }; - - public static void append(long hash, int bits, CharSink sink) { - if (hash == GeoHashes.NULL) { - sink.putAscii("null"); - } else { - sink.putAscii('\"'); - if (bits < 0) { - GeoHashes.appendCharsUnsafe(hash, -bits, sink); - } else { - GeoHashes.appendBinaryStringUnsafe(hash, bits, sink); - } - sink.putAscii('\"'); - } - } - - public static void appendBinaryStringUnsafe(long hash, int bits, CharSink sink) { - // Below assertion can happen if there is corrupt metadata - // which should not happen in production code since reader and writer check table metadata - assert bits > 0 && bits <= ColumnType.GEOLONG_MAX_BITS; - for (int i = bits - 1; i >= 0; --i) { - sink.putAscii(((hash >> i) & 1) == 1 ? '1' : '0'); - } - } - - public static void appendChars(long hash, int chars, CharSink sink) { - if (hash != NULL) { - appendCharsUnsafe(hash, chars, sink); - } - } - - public static void appendCharsUnsafe(long hash, int chars, CharSink sink) { - // Below assertion can happen if there is corrupt metadata - // which should not happen in production code since reader and writer check table metadata - assert chars > 0 && chars <= MAX_STRING_LENGTH; - for (int i = chars - 1; i >= 0; --i) { - sink.putAscii(base32[(int) ((hash >> i * 5) & 0x1F)]); - } - } - - public static long fromCoordinatesDeg(double lat, double lon, int bits) throws NumericException { - if (lat < -90.0 || lat > 90.0) { - throw NumericException.instance(); - } - if (lon < -180.0 || lon > 180.0) { - throw NumericException.instance(); - } - if (bits < 0 || bits > ColumnType.GEOLONG_MAX_BITS) { - throw NumericException.instance(); - } - return fromCoordinatesDegUnsafe(lat, lon, bits); - } - - public static long fromCoordinatesDegUnsafe(double lat, double lon, int bits) { - long latq = (long) Math.scalb((lat + 90.0) / 180.0, 32); - long lngq = (long) Math.scalb((lon + 180.0) / 360.0, 32); - return Numbers.interleaveBits(latq, lngq) >>> (64 - bits); - } -} diff --git a/core/src/main/java/io/questdb/client/cairo/TableUtils.java b/core/src/main/java/io/questdb/client/cairo/TableUtils.java index def4f02..f250e9f 100644 --- a/core/src/main/java/io/questdb/client/cairo/TableUtils.java +++ b/core/src/main/java/io/questdb/client/cairo/TableUtils.java @@ -24,8 +24,10 @@ package io.questdb.client.cairo; +import org.jetbrains.annotations.NotNull; + public final class TableUtils { - public static boolean isValidColumnName(CharSequence columnName, int fsFileNameLimit) { + public static boolean isValidColumnName(@NotNull CharSequence columnName, int fsFileNameLimit) { final int length = columnName.length(); if (length > fsFileNameLimit) { // Most file systems do not support file names longer than 255 bytes @@ -133,4 +135,4 @@ public static boolean isValidTableName(CharSequence tableName, int fsFileNameLim } return length > 0 && tableName.charAt(0) != ' ' && tableName.charAt(length - 1) != ' '; } -} \ No newline at end of file +} diff --git a/core/src/main/java/io/questdb/client/cutlass/http/HttpCookie.java b/core/src/main/java/io/questdb/client/cutlass/http/HttpCookie.java deleted file mode 100644 index 8ac55c9..0000000 --- a/core/src/main/java/io/questdb/client/cutlass/http/HttpCookie.java +++ /dev/null @@ -1,82 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.cutlass.http; - -import io.questdb.client.std.Mutable; -import io.questdb.client.std.str.CharSink; -import io.questdb.client.std.str.DirectUtf8String; -import io.questdb.client.std.str.Sinkable; -import org.jetbrains.annotations.NotNull; - -public class HttpCookie implements Mutable, Sinkable { - public DirectUtf8String cookieName; - public DirectUtf8String domain; - public long expires = -1L; - public boolean httpOnly; - public long maxAge; - public boolean partitioned; - public DirectUtf8String path; - public DirectUtf8String sameSite; - public boolean secure; - public DirectUtf8String value; - - @Override - public void clear() { - this.domain = null; - this.expires = -1L; - this.httpOnly = false; - this.maxAge = 0L; - this.partitioned = false; - this.path = null; - this.sameSite = null; - this.secure = false; - this.value = null; - this.cookieName = null; - } - - @Override - public void toSink(@NotNull CharSink sink) { - sink.put('{'); - - sink.put("cookieName=").putQuoted(cookieName); - sink.put(", value=").putQuoted(value); - if (domain != null) { - sink.put(", domain=").putQuoted(domain); - } - if (path != null) { - sink.put(", path=").putQuoted(path); - } - sink.put(", secure=").put(secure); - sink.put(", httpOnly=").put(httpOnly); - sink.put(", partitioned=").put(partitioned); - sink.put(", expires=").put(expires); - sink.put(", maxAge=").put(maxAge); - if (sameSite != null) { - sink.put(", sameSite=").putQuoted(sameSite); - } - sink.put('}'); - } - -} \ No newline at end of file diff --git a/core/src/main/java/io/questdb/client/cutlass/http/HttpHeaderParameterValue.java b/core/src/main/java/io/questdb/client/cutlass/http/HttpHeaderParameterValue.java deleted file mode 100644 index f18f4c4..0000000 --- a/core/src/main/java/io/questdb/client/cutlass/http/HttpHeaderParameterValue.java +++ /dev/null @@ -1,46 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.cutlass.http; - -import io.questdb.client.std.str.DirectUtf8String; - -public class HttpHeaderParameterValue { - private long hi; - private DirectUtf8String str; - - public long getHi() { - return hi; - } - - public DirectUtf8String getStr() { - return str; - } - - public HttpHeaderParameterValue of(long hi, DirectUtf8String str) { - this.hi = hi; - this.str = str; - return this; - } -} \ No newline at end of file diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/HttpClientException.java b/core/src/main/java/io/questdb/client/cutlass/http/client/HttpClientException.java index 4b2c7ab..82515dd 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/HttpClientException.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/HttpClientException.java @@ -31,6 +31,7 @@ public class HttpClientException extends RuntimeException { private final StringSink message = new StringSink(); private int errno = Integer.MIN_VALUE; + private boolean isTimeout; public HttpClientException(String message) { this.message.put(message); @@ -53,6 +54,10 @@ public String getMessage() { return errNoRender + " " + message; } + public boolean isTimeout() { + return isTimeout; + } + public HttpClientException put(char value) { message.put(value); return this; @@ -77,4 +82,9 @@ public HttpClientException putSize(long value) { message.putSize(value); return this; } + + public HttpClientException flagAsTimeout() { + this.isTimeout = true; + return this; + } } \ No newline at end of file diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java new file mode 100644 index 0000000..6e38872 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -0,0 +1,916 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.http.client; + +import io.questdb.client.HttpClientConfiguration; +import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; +import io.questdb.client.cutlass.qwp.websocket.WebSocketFrameParser; +import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode; +import io.questdb.client.network.IOOperation; +import io.questdb.client.network.NetworkFacade; +import io.questdb.client.network.Socket; +import io.questdb.client.network.SocketFactory; +import io.questdb.client.network.TlsSessionInitFailedException; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Misc; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.SecureRnd; +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.Vect; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Base64; + +import static java.util.concurrent.TimeUnit.NANOSECONDS; + +/** + * Zero-GC WebSocket client built on QuestDB's native socket infrastructure. + *

+ * This client uses native memory buffers and non-blocking I/O with + * platform-specific event notification (epoll/kqueue/select). + *

+ * Features: + *

    + *
  • Zero-copy send path using {@link WebSocketSendBuffer}
    • + *
  • Automatic ping/pong handling
    • + *
  • TLS support
    • + *
  • Connection keep-alive
    • + *
+ *

+ * Thread safety: This class is NOT thread-safe. Each connection should be + * accessed from a single thread at a time. + */ +public abstract class WebSocketClient implements QuietCloseable { + + private static final int DEFAULT_RECV_BUFFER_SIZE = 65536; + private static final int DEFAULT_SEND_BUFFER_SIZE = 65536; + private static final Logger LOG = LoggerFactory.getLogger(WebSocketClient.class); + private static final ThreadLocal SHA1_DIGEST = ThreadLocal.withInitial(() -> { + try { + return MessageDigest.getInstance("SHA-1"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("SHA-1 not available", e); + } + }); + private static final byte[] WEBSOCKET_GUID_BYTES = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11".getBytes(StandardCharsets.US_ASCII); + protected final NetworkFacade nf; + protected final Socket socket; + private final WebSocketSendBuffer controlFrameBuffer; + private final int defaultTimeout; + private final WebSocketFrameParser frameParser; + private final int maxRecvBufSize; + private final SecureRnd rnd; + private final WebSocketSendBuffer sendBuffer; + private boolean closed; + private int fragmentBufPos; + private long fragmentBufPtr; // native buffer for accumulating fragment payloads + private int fragmentBufSize; + // Fragmentation state (RFC 6455 Section 5.4) + private int fragmentOpcode = -1; // opcode of first fragment, -1 = not in a fragmented message + // Handshake key for verification + private String handshakeKey; + // Connection state + private CharSequence host; + private int port; + // Receive buffer (native memory) + private long recvBufPtr; + private int recvBufSize; + private int recvPos; // Write position + private int recvReadPos; // Read position + private boolean upgraded; + + public WebSocketClient(HttpClientConfiguration configuration, SocketFactory socketFactory) { + this.nf = configuration.getNetworkFacade(); + this.socket = socketFactory.newInstance(nf, LOG); + this.defaultTimeout = configuration.getTimeout(); + + int sendBufSize = Math.max(configuration.getInitialRequestBufferSize(), DEFAULT_SEND_BUFFER_SIZE); + int maxSendBufSize = Math.max(configuration.getMaximumRequestBufferSize(), sendBufSize); + WebSocketSendBuffer sendBuf = null; + WebSocketSendBuffer controlBuf = null; + try { + sendBuf = new WebSocketSendBuffer(sendBufSize, maxSendBufSize); + // Control frames (ping/pong/close) have max 125-byte payload + 14-byte header. + // This dedicated buffer prevents sendPongFrame from clobbering an in-progress + // frame being built in the main sendBuffer. + controlBuf = new WebSocketSendBuffer(256, 256); + + this.recvBufSize = Math.max(configuration.getResponseBufferSize(), DEFAULT_RECV_BUFFER_SIZE); + this.maxRecvBufSize = Math.max(configuration.getMaximumResponseBufferSize(), recvBufSize); + this.recvBufPtr = Unsafe.malloc(recvBufSize, MemoryTag.NATIVE_DEFAULT); + } catch (Throwable t) { + Misc.free(controlBuf); + Misc.free(sendBuf); + Misc.free(socket); + throw t; + } + this.sendBuffer = sendBuf; + this.controlFrameBuffer = controlBuf; + this.recvPos = 0; + this.recvReadPos = 0; + + this.frameParser = new WebSocketFrameParser(); + this.rnd = new SecureRnd(); + this.upgraded = false; + this.closed = false; + } + + @Override + public void close() { + if (!closed) { + closed = true; + + // Try to send close frame + if (upgraded && !socket.isClosed()) { + try { + sendCloseFrame(WebSocketCloseCode.NORMAL_CLOSURE, null, 1000); + } catch (Exception e) { + // Ignore errors during close + } + } + + disconnect(); + sendBuffer.close(); + controlFrameBuffer.close(); + + if (fragmentBufPtr != 0) { + Unsafe.free(fragmentBufPtr, fragmentBufSize, MemoryTag.NATIVE_DEFAULT); + fragmentBufPtr = 0; + } + + if (recvBufPtr != 0) { + Unsafe.free(recvBufPtr, recvBufSize, MemoryTag.NATIVE_DEFAULT); + recvBufPtr = 0; + } + } + } + + /** + * Connects to a WebSocket server. + * + * @param host the server hostname + * @param port the server port + * @param timeout connection timeout in milliseconds + */ + public void connect(CharSequence host, int port, int timeout) { + if (closed) { + throw new HttpClientException("WebSocket client is closed"); + } + + // Close existing connection if connecting to different host:port + if (this.host != null && (!this.host.equals(host) || this.port != port)) { + disconnect(); + } + + if (socket.isClosed()) { + doConnect(host, port, timeout); + } + + this.host = host; + this.port = port; + } + + /** + * Connects using default timeout. + */ + public void connect(CharSequence host, int port) { + connect(host, port, defaultTimeout); + } + + /** + * Disconnects the socket without closing the client. + * The client can be reconnected by calling connect() again. + */ + public void disconnect() { + Misc.free(socket); + upgraded = false; + host = null; + port = 0; + recvPos = 0; + recvReadPos = 0; + resetFragmentState(); + } + + /** + * Returns the connected host. + */ + public CharSequence getHost() { + return host; + } + + /** + * Returns the connected port. + */ + public int getPort() { + return port; + } + + /** + * Gets the send buffer for building WebSocket frames. + *

+ * Usage: + * 

+     * WebSocketSendBuffer buf = client.getSendBuffer();
+     * buf.beginBinaryFrame();
+     * buf.putLong(data);
+     * WebSocketSendBuffer.FrameInfo frame = buf.endBinaryFrame();
+     * client.sendFrame(frame, timeout);
+     * buf.reset();
+     *
+ */ + public WebSocketSendBuffer getSendBuffer() { + return sendBuffer; + } + + /** + * Returns whether the WebSocket is connected and upgraded. + */ + public boolean isConnected() { + return upgraded && !closed && !socket.isClosed(); + } + + /** + * Receives and processes WebSocket frames. + * + * @param handler frame handler callback + * @param timeout timeout in milliseconds + * @return true if a frame was received, false on timeout + */ + public boolean receiveFrame(WebSocketFrameHandler handler, int timeout) { + checkConnected(); + + // First, try to parse any data already in buffer + Boolean result = tryParseFrame(handler); + if (result != null) { + return result; + } + + // Need more data + long startTime = System.nanoTime(); + while (true) { + int remainingTimeout = remainingTime(timeout, startTime); + if (remainingTimeout <= 0) { + return false; // Timeout + } + + // Ensure buffer has space + if (recvPos >= recvBufSize - 1024) { + growRecvBuffer(); + } + + int bytesRead = recvOrTimeout(recvBufPtr + recvPos, recvBufSize - recvPos, remainingTimeout); + if (bytesRead <= 0) { + return false; // Timeout + } + recvPos += bytesRead; + + result = tryParseFrame(handler); + if (result != null) { + return result; + } + } + } + + /** + * Sends binary data as a WebSocket binary frame. + * + * @param dataPtr pointer to data + * @param length data length + * @param timeout timeout in milliseconds + */ + public void sendBinary(long dataPtr, int length, int timeout) { + checkConnected(); + sendBuffer.reset(); + sendBuffer.beginFrame(); + sendBuffer.putBlockOfBytes(dataPtr, length); + WebSocketSendBuffer.FrameInfo frame = sendBuffer.endBinaryFrame(); + doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout); + sendBuffer.reset(); + } + + /** + * Sends binary data with default timeout. + */ + public void sendBinary(long dataPtr, int length) { + sendBinary(dataPtr, length, defaultTimeout); + } + + /** + * Sends a close frame. + */ + public void sendCloseFrame(int code, String reason, int timeout) { + controlFrameBuffer.reset(); + WebSocketSendBuffer.FrameInfo frame = controlFrameBuffer.writeCloseFrame(code, reason); + try { + doSend(controlFrameBuffer.getBufferPtr() + frame.offset, frame.length, timeout); + } finally { + controlFrameBuffer.reset(); + } + } + + /** + * Sends a ping frame. + */ + public void sendPing(int timeout) { + checkConnected(); + controlFrameBuffer.reset(); + WebSocketSendBuffer.FrameInfo frame = controlFrameBuffer.writePingFrame(); + doSend(controlFrameBuffer.getBufferPtr() + frame.offset, frame.length, timeout); + controlFrameBuffer.reset(); + } + + /** + * Non-blocking attempt to receive a WebSocket frame. + * Returns immediately if no complete frame is available. + * + * @param handler frame handler callback + * @return true if a frame was received, false if no data available + */ + public boolean tryReceiveFrame(WebSocketFrameHandler handler) { + checkConnected(); + + // First, try to parse any data already in buffer + Boolean result = tryParseFrame(handler); + if (result != null) { + return result; + } + + // Try one non-blocking recv + if (recvPos >= recvBufSize - 1024) { + growRecvBuffer(); + } + + int n = socket.recv(recvBufPtr + recvPos, recvBufSize - recvPos); + if (n < 0) { + throw new HttpClientException("peer disconnect [errno=").errno(nf.errno()).put(']'); + } + if (n == 0) { + return false; // No data available + } + recvPos += n; + + // Try to parse again + result = tryParseFrame(handler); + return result != null && result; + } + + /** + * Performs WebSocket upgrade handshake. + * + * @param path the WebSocket endpoint path (e.g., "/ws") + * @param timeout timeout in milliseconds + * @param authorizationHeader the Authorization header value (e.g., "Basic ..."), or null + */ + public void upgrade(CharSequence path, int timeout, CharSequence authorizationHeader) { + if (closed) { + throw new HttpClientException("WebSocket client is closed"); + } + if (socket.isClosed()) { + throw new HttpClientException("Not connected"); + } + if (upgraded) { + return; // Already upgraded + } + + // Generate random key + byte[] keyBytes = new byte[16]; + for (int i = 0; i < 16; i++) { + keyBytes[i] = (byte) rnd.nextInt(); + } + handshakeKey = Base64.getEncoder().encodeToString(keyBytes); + + // Build upgrade request + sendBuffer.reset(); + sendBuffer.putAscii("GET "); + sendBuffer.putAscii(path); + sendBuffer.putAscii(" HTTP/1.1\r\n"); + sendBuffer.putAscii("Host: "); + sendBuffer.putAscii(host); + if ((socket.supportsTls() && port != 443) || (!socket.supportsTls() && port != 80)) { + sendBuffer.putAscii(":"); + sendBuffer.putAscii(Integer.toString(port)); + } + sendBuffer.putAscii("\r\n"); + sendBuffer.putAscii("Upgrade: websocket\r\n"); + sendBuffer.putAscii("Connection: Upgrade\r\n"); + sendBuffer.putAscii("Sec-WebSocket-Key: "); + sendBuffer.putAscii(handshakeKey); + sendBuffer.putAscii("\r\n"); + sendBuffer.putAscii("Sec-WebSocket-Version: 13\r\n"); + if (authorizationHeader != null) { + sendBuffer.putAscii("Authorization: "); + sendBuffer.putAscii(authorizationHeader); + sendBuffer.putAscii("\r\n"); + } + sendBuffer.putAscii("\r\n"); + + // Send request + long startTime = System.nanoTime(); + doSend(sendBuffer.getBufferPtr(), sendBuffer.getWritePos(), timeout); + + // Read response + int remainingTimeout = getRemainingTimeOrThrow(timeout, startTime); + readUpgradeResponse(remainingTimeout); + + upgraded = true; + sendBuffer.reset(); + LOG.debug("WebSocket upgraded [path={}]", path); + } + + /** + * Performs upgrade with default timeout. + */ + public void upgrade(CharSequence path) { + upgrade(path, defaultTimeout, null); + } + + /** + * Performs upgrade with default timeout and authorization header. + */ + public void upgrade(CharSequence path, CharSequence authorizationHeader) { + upgrade(path, defaultTimeout, authorizationHeader); + } + + /** + * Performs upgrade without authorization header. + */ + public void upgrade(CharSequence path, int timeout) { + upgrade(path, timeout, null); + } + + private static String computeAcceptKey(String key) { + MessageDigest sha1 = SHA1_DIGEST.get(); + sha1.reset(); + for (int i = 0, n = key.length(); i < n; i++) { + sha1.update((byte) key.charAt(i)); + } + sha1.update(WEBSOCKET_GUID_BYTES); + return Base64.getEncoder().encodeToString(sha1.digest()); + } + + private static boolean containsHeaderValue(String response, String headerName, String expectedValue, boolean ignoreValueCase) { + int headerLen = headerName.length(); + int responseLen = response.length(); + for (int i = 0; i <= responseLen - headerLen; i++) { + if (response.regionMatches(true, i, headerName, 0, headerLen)) { + int valueStart = i + headerLen; + int lineEnd = response.indexOf('\r', valueStart); + if (lineEnd < 0) { + lineEnd = responseLen; + } + String actualValue = response.substring(valueStart, lineEnd).trim(); + return ignoreValueCase + ? actualValue.equalsIgnoreCase(expectedValue) + : actualValue.equals(expectedValue); + } + } + return false; + } + + private static int remainingTime(int timeoutMillis, long startTimeNanos) { + return timeoutMillis - (int) NANOSECONDS.toMillis(System.nanoTime() - startTimeNanos); + } + + private void appendToFragmentBuffer(long payloadPtr, int payloadLen) { + if (payloadLen == 0) { + return; + } + int required = fragmentBufPos + payloadLen; + if (required > maxRecvBufSize) { + throw new HttpClientException("WebSocket fragment buffer size exceeded maximum [required=") + .put(required) + .put(", max=") + .put(maxRecvBufSize) + .put(']'); + } + if (fragmentBufPtr == 0) { + fragmentBufSize = Math.max(required, DEFAULT_RECV_BUFFER_SIZE); + fragmentBufPtr = Unsafe.malloc(fragmentBufSize, MemoryTag.NATIVE_DEFAULT); + } else if (required > fragmentBufSize) { + int newSize = (int) Math.min(Math.max((long) fragmentBufSize * 2, required), maxRecvBufSize); + fragmentBufPtr = Unsafe.realloc(fragmentBufPtr, fragmentBufSize, newSize, MemoryTag.NATIVE_DEFAULT); + fragmentBufSize = newSize; + } + Vect.memmove(fragmentBufPtr + fragmentBufPos, payloadPtr, payloadLen); + fragmentBufPos += payloadLen; + } + + private void checkConnected() { + if (closed) { + throw new HttpClientException("WebSocket client is closed"); + } + if (!upgraded) { + throw new HttpClientException("WebSocket not connected or upgraded"); + } + } + + private void compactRecvBuffer() { + if (recvReadPos > 0) { + int remaining = recvPos - recvReadPos; + if (remaining > 0) { + Vect.memmove(recvBufPtr, recvBufPtr + recvReadPos, remaining); + } + recvPos = remaining; + recvReadPos = 0; + } + } + + private int dieIfNegative(int byteCount) { + if (byteCount < 0) { + throw new HttpClientException("peer disconnect [errno=").errno(nf.errno()).put(']'); + } + return byteCount; + } + + private void doConnect(CharSequence host, int port, int timeout) { + int fd = nf.socketTcp(true); + if (fd < 0) { + throw new HttpClientException("could not allocate a file descriptor [errno=").errno(nf.errno()).put(']'); + } + + if (nf.setTcpNoDelay(fd, true) < 0) { + LOG.info("could not disable Nagle's algorithm [fd={}, errno={}]", fd, nf.errno()); + } + + socket.of(fd); + nf.configureKeepAlive(fd); + + long addrInfo = nf.getAddrInfo(host, port); + if (addrInfo == -1) { + disconnect(); + throw new HttpClientException("could not resolve host [host=").put(host).put(']'); + } + + if (nf.connectAddrInfo(fd, addrInfo) != 0) { + int errno = nf.errno(); + nf.freeAddrInfo(addrInfo); + disconnect(); + throw new HttpClientException("could not connect [host=").put(host) + .put(", port=").put(port) + .put(", errno=").put(errno).put(']'); + } + nf.freeAddrInfo(addrInfo); + + if (nf.configureNonBlocking(fd) < 0) { + int errno = nf.errno(); + disconnect(); + throw new HttpClientException("could not configure non-blocking [fd=").put(fd) + .put(", errno=").put(errno).put(']'); + } + + if (socket.supportsTls()) { + try { + socket.startTlsSession(host); + } catch (TlsSessionInitFailedException e) { + int errno = nf.errno(); + disconnect(); + throw new HttpClientException("could not start TLS session [fd=").put(fd) + .put(", error=").put(e.getFlyweightMessage()) + .put(", errno=").put(errno).put(']'); + } + } + + setupIoWait(); + LOG.debug("Connected to [host={}, port={}]", host, port); + } + + private void doSend(long ptr, int len, int timeout) { + long startTime = System.nanoTime(); + while (len > 0) { + int remainingTimeout = getRemainingTimeOrThrow(timeout, startTime); + ioWait(remainingTimeout, IOOperation.WRITE); + int sent = dieIfNegative(socket.send(ptr, len)); + while (socket.wantsTlsWrite()) { + remainingTimeout = getRemainingTimeOrThrow(timeout, startTime); + ioWait(remainingTimeout, IOOperation.WRITE); + dieIfNegative(socket.tlsIO(Socket.WRITE_FLAG)); + } + if (sent > 0) { + ptr += sent; + len -= sent; + } + } + } + + private int findHeaderEnd() { + // Look for \r\n\r\n + for (int i = 0; i < recvPos - 3; i++) { + if (Unsafe.getUnsafe().getByte(recvBufPtr + i) == '\r' && + Unsafe.getUnsafe().getByte(recvBufPtr + i + 1) == '\n' && + Unsafe.getUnsafe().getByte(recvBufPtr + i + 2) == '\r' && + Unsafe.getUnsafe().getByte(recvBufPtr + i + 3) == '\n') { + return i + 4; + } + } + return -1; + } + + private int getRemainingTimeOrThrow(int timeoutMillis, long startTimeNanos) { + int remaining = remainingTime(timeoutMillis, startTimeNanos); + if (remaining <= 0) { + throw new HttpClientException("timed out [errno=").errno(nf.errno()).put(']').flagAsTimeout(); + } + return remaining; + } + + private void growRecvBuffer() { + int newSize = (int) Math.min((long) recvBufSize * 2, maxRecvBufSize); + if (newSize >= maxRecvBufSize) { + if (recvBufSize >= maxRecvBufSize) { + throw new HttpClientException("WebSocket receive buffer size exceeded maximum [current=") + .put(recvBufSize) + .put(", max=") + .put(maxRecvBufSize) + .put(']'); + } + newSize = maxRecvBufSize; + } + recvBufPtr = Unsafe.realloc(recvBufPtr, recvBufSize, newSize, MemoryTag.NATIVE_DEFAULT); + recvBufSize = newSize; + } + + private void readUpgradeResponse(int timeout) { + // Read HTTP response into receive buffer + long startTime = System.nanoTime(); + + while (true) { + int remainingTimeout = getRemainingTimeOrThrow(timeout, startTime); + int bytesRead = recvOrDie(recvBufPtr + recvPos, recvBufSize - recvPos, remainingTimeout); + if (bytesRead > 0) { + recvPos += bytesRead; + } + + // Check for end of headers (\r\n\r\n) + int headerEnd = findHeaderEnd(); + if (headerEnd > 0) { + validateUpgradeResponse(headerEnd); + // Compact buffer - move remaining data to start + int remaining = recvPos - headerEnd; + if (remaining > 0) { + Vect.memmove(recvBufPtr, recvBufPtr + headerEnd, remaining); + } + recvPos = remaining; + recvReadPos = 0; + return; + } + + if (recvPos >= recvBufSize) { + throw new HttpClientException("HTTP response too large"); + } + } + } + + private int recvOrDie(long ptr, int len, int timeout) { + long startTime = System.nanoTime(); + int n = dieIfNegative(socket.recv(ptr, len)); + if (n == 0) { + ioWait(getRemainingTimeOrThrow(timeout, startTime), IOOperation.READ); + n = dieIfNegative(socket.recv(ptr, len)); + } + return n; + } + + private int recvOrTimeout(long ptr, int len, int timeout) { + int n = socket.recv(ptr, len); + if (n < 0) { + throw new HttpClientException("peer disconnect [errno=").errno(nf.errno()).put(']'); + } + if (n == 0) { + try { + ioWait(timeout, IOOperation.READ); + } catch (HttpClientException e) { + if (!e.isTimeout()) { + throw e; + } + return 0; + } + n = socket.recv(ptr, len); + if (n < 0) { + throw new HttpClientException("peer disconnect [errno=").errno(nf.errno()).put(']'); + } + } + return n; + } + + private void resetFragmentState() { + fragmentOpcode = -1; + fragmentBufPos = 0; + } + + private void sendCloseFrameEcho(int code) { + try { + controlFrameBuffer.reset(); + WebSocketSendBuffer.FrameInfo frame = controlFrameBuffer.writeCloseFrame(code, null); + doSend(controlFrameBuffer.getBufferPtr() + frame.offset, frame.length, 1000); + controlFrameBuffer.reset(); + } catch (Exception e) { + LOG.error("Failed to echo close frame: {}", e.getMessage()); + } + } + + private void sendPongFrame(long payloadPtr, int payloadLen) { + try { + controlFrameBuffer.reset(); + WebSocketSendBuffer.FrameInfo frame = controlFrameBuffer.writePongFrame(payloadPtr, payloadLen); + doSend(controlFrameBuffer.getBufferPtr() + frame.offset, frame.length, 1000); + controlFrameBuffer.reset(); + } catch (Exception e) { + LOG.error("Failed to send pong: {}", e.getMessage()); + } + } + + private Boolean tryParseFrame(WebSocketFrameHandler handler) { + if (recvPos <= recvReadPos) { + return null; // No data + } + + frameParser.reset(); + int consumed = frameParser.parse(recvBufPtr + recvReadPos, recvBufPtr + recvPos); + + if (frameParser.getState() == WebSocketFrameParser.STATE_NEED_MORE || + frameParser.getState() == WebSocketFrameParser.STATE_NEED_PAYLOAD) { + return null; // Need more data + } + + if (frameParser.getState() == WebSocketFrameParser.STATE_ERROR) { + throw new HttpClientException("WebSocket frame parse error: ") + .put(WebSocketCloseCode.describe(frameParser.getErrorCode())); + } + + if (frameParser.getState() == WebSocketFrameParser.STATE_COMPLETE) { + long payloadPtr = recvBufPtr + recvReadPos + frameParser.getHeaderSize(); + long payloadLength = frameParser.getPayloadLength(); + if (payloadLength > Integer.MAX_VALUE) { + throw new HttpClientException("WebSocket frame payload too large [length=") + .put(payloadLength).put(']'); + } + int payloadLen = (int) payloadLength; + + // Handle frame by opcode + int opcode = frameParser.getOpcode(); + switch (opcode) { + case WebSocketOpcode.PING: + // Auto-respond with pong + sendPongFrame(payloadPtr, payloadLen); + if (handler != null) { + handler.onPing(payloadPtr, payloadLen); + } + break; + case WebSocketOpcode.PONG: + if (handler != null) { + handler.onPong(payloadPtr, payloadLen); + } + break; + case WebSocketOpcode.CLOSE: + int closeCode = 0; + String reason = null; + if (payloadLen >= 2) { + closeCode = ((Unsafe.getUnsafe().getByte(payloadPtr) & 0xFF) << 8) + | (Unsafe.getUnsafe().getByte(payloadPtr + 1) & 0xFF); + if (payloadLen > 2) { + byte[] reasonBytes = new byte[payloadLen - 2]; + for (int i = 0; i < reasonBytes.length; i++) { + reasonBytes[i] = Unsafe.getUnsafe().getByte(payloadPtr + 2 + i); + } + reason = new String(reasonBytes, StandardCharsets.UTF_8); + } + } + // RFC 6455 Section 5.5.1: echo a close frame back before + // marking the connection as no longer upgraded + sendCloseFrameEcho(closeCode); + upgraded = false; + if (handler != null) { + handler.onClose(closeCode, reason); + } + break; + case WebSocketOpcode.BINARY: + case WebSocketOpcode.TEXT: + if (frameParser.isFin()) { + if (fragmentOpcode != -1) { + throw new HttpClientException("WebSocket protocol error: new data frame during fragmented message"); + } + if (handler != null) { + if (opcode == WebSocketOpcode.BINARY) { + handler.onBinaryMessage(payloadPtr, payloadLen); + } else { + handler.onTextMessage(payloadPtr, payloadLen); + } + } + } else { + if (fragmentOpcode != -1) { + throw new HttpClientException("WebSocket protocol error: new data frame during fragmented message"); + } + fragmentOpcode = opcode; + appendToFragmentBuffer(payloadPtr, payloadLen); + } + break; + case WebSocketOpcode.CONTINUATION: + if (fragmentOpcode == -1) { + throw new HttpClientException("WebSocket protocol error: continuation frame without initial fragment"); + } + appendToFragmentBuffer(payloadPtr, payloadLen); + if (frameParser.isFin()) { + if (handler != null) { + if (fragmentOpcode == WebSocketOpcode.BINARY) { + handler.onBinaryMessage(fragmentBufPtr, fragmentBufPos); + } else { + handler.onTextMessage(fragmentBufPtr, fragmentBufPos); + } + } + resetFragmentState(); + } + break; + } + + // Advance read position + recvReadPos += consumed; + + // Compact buffer if needed + compactRecvBuffer(); + + return true; + } + + return false; + } + + private void validateUpgradeResponse(int headerEnd) { + // Extract response as string for parsing + byte[] responseBytes = new byte[headerEnd]; + for (int i = 0; i < headerEnd; i++) { + responseBytes[i] = Unsafe.getUnsafe().getByte(recvBufPtr + i); + } + String response = new String(responseBytes, StandardCharsets.US_ASCII); + + // Check status line + if (!response.startsWith("HTTP/1.1 101")) { + String statusLine = response.split("\r\n")[0]; + throw new HttpClientException("WebSocket upgrade failed: ").put(statusLine); + } + + // Verify Upgrade: websocket (case-insensitive value per RFC 6455 Section 4.1) + if (!containsHeaderValue(response, "Upgrade:", "websocket", true)) { + throw new HttpClientException("Missing or invalid Upgrade header in WebSocket response"); + } + + // Verify Connection: Upgrade (case-insensitive value per RFC 6455 Section 4.1) + if (!containsHeaderValue(response, "Connection:", "Upgrade", true)) { + throw new HttpClientException("Missing or invalid Connection header in WebSocket response"); + } + + // Verify Sec-WebSocket-Accept (exact value match per RFC 6455 Section 4.1) + String expectedAccept = computeAcceptKey(handshakeKey); + if (!containsHeaderValue(response, "Sec-WebSocket-Accept:", expectedAccept, false)) { + throw new HttpClientException("Invalid Sec-WebSocket-Accept header"); + } + } + + protected void dieWaiting(int n) { + if (n == 1) { + return; + } + if (n == 0) { + throw new HttpClientException("timed out [errno=").put(nf.errno()).put(']').flagAsTimeout(); + } + throw new HttpClientException("queue error [errno=").put(nf.errno()).put(']'); + } + + /** + * Waits for I/O readiness using platform-specific mechanism. + * + * @param timeout timeout in milliseconds + * @param op I/O operation (READ or WRITE) + */ + protected abstract void ioWait(int timeout, int op); + + /** + * Sets up platform-specific I/O wait mechanism after connection. + */ + protected abstract void setupIoWait(); +} diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientFactory.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientFactory.java new file mode 100644 index 0000000..f48edd9 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientFactory.java @@ -0,0 +1,141 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.http.client; + +import io.questdb.client.ClientTlsConfiguration; +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.HttpClientConfiguration; +import io.questdb.client.network.JavaTlsClientSocketFactory; +import io.questdb.client.network.PlainSocketFactory; +import io.questdb.client.network.SocketFactory; +import io.questdb.client.std.Os; + +/** + * Factory for creating platform-specific {@link WebSocketClient} instances. + *

+ * Usage: + * 

+ * // Plain text connection
+ * WebSocketClient client = WebSocketClientFactory.newPlainTextInstance();
+ *
+ * // TLS connection
+ * WebSocketClient client = WebSocketClientFactory.newTlsInstance(config, tlsConfig);
+ *
+ * // Connect and upgrade
+ * client.connect("localhost", 9000);
+ * client.upgrade("/ws");
+ *
+ * // Send data
+ * WebSocketSendBuffer buf = client.getSendBuffer();
+ * buf.beginBinaryFrame();
+ * buf.putLong(data);
+ * WebSocketSendBuffer.FrameInfo frame = buf.endBinaryFrame();
+ * client.sendFrame(frame);
+ * buf.reset();
+ *
+ * // Receive data
+ * client.receiveFrame(handler);
+ *
+ * client.close();
+ *
+ */ +public class WebSocketClientFactory { + + // Utility class -- no instantiation + private WebSocketClientFactory() { + } + + /** + * Creates a new WebSocket client with insecure TLS (no certificate validation). + *

+ * WARNING: Only use this for testing. Production code should use proper TLS validation. + * + * @return a new WebSocket client with insecure TLS + */ + public static WebSocketClient newInsecureTlsInstance() { + return newInstance(DefaultHttpClientConfiguration.INSTANCE, JavaTlsClientSocketFactory.INSECURE_NO_VALIDATION); + } + + /** + * Creates a new WebSocket client with the specified configuration and socket factory. + * + * @param configuration the HTTP client configuration + * @param socketFactory the socket factory for creating sockets + * @return a new platform-specific WebSocket client + */ + public static WebSocketClient newInstance(HttpClientConfiguration configuration, SocketFactory socketFactory) { + switch (Os.type) { + case Os.LINUX: + return new WebSocketClientLinux(configuration, socketFactory); + case Os.DARWIN: + case Os.FREEBSD: + return new WebSocketClientOsx(configuration, socketFactory); + case Os.WINDOWS: + return new WebSocketClientWindows(configuration, socketFactory); + default: + throw new UnsupportedOperationException("Unsupported platform: " + Os.type); + } + } + + /** + * Creates a new plain text WebSocket client with default configuration. + * + * @return a new plain text WebSocket client + */ + public static WebSocketClient newPlainTextInstance() { + return newPlainTextInstance(DefaultHttpClientConfiguration.INSTANCE); + } + + /** + * Creates a new plain text WebSocket client with the specified configuration. + * + * @param configuration the HTTP client configuration + * @return a new plain text WebSocket client + */ + public static WebSocketClient newPlainTextInstance(HttpClientConfiguration configuration) { + return newInstance(configuration, PlainSocketFactory.INSTANCE); + } + + /** + * Creates a new TLS WebSocket client with the specified configuration. + * + * @param configuration the HTTP client configuration + * @param tlsConfig the TLS configuration + * @return a new TLS WebSocket client + */ + public static WebSocketClient newTlsInstance(HttpClientConfiguration configuration, ClientTlsConfiguration tlsConfig) { + return newInstance(configuration, new JavaTlsClientSocketFactory(tlsConfig)); + } + + /** + * Creates a new TLS WebSocket client with default HTTP configuration. + * + * @param tlsConfig the TLS configuration + * @return a new TLS WebSocket client + */ + public static WebSocketClient newTlsInstance(ClientTlsConfiguration tlsConfig) { + return newTlsInstance(DefaultHttpClientConfiguration.INSTANCE, tlsConfig); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientLinux.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientLinux.java new file mode 100644 index 0000000..0533a10 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientLinux.java @@ -0,0 +1,76 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.http.client; + +import io.questdb.client.HttpClientConfiguration; +import io.questdb.client.network.Epoll; +import io.questdb.client.network.EpollAccessor; +import io.questdb.client.network.IOOperation; +import io.questdb.client.network.SocketFactory; +import io.questdb.client.std.Misc; + +/** + * Linux-specific WebSocket client using epoll for I/O waiting. + */ +public class WebSocketClientLinux extends WebSocketClient { + private Epoll epoll; + + public WebSocketClientLinux(HttpClientConfiguration configuration, SocketFactory socketFactory) { + super(configuration, socketFactory); + try { + epoll = new Epoll( + configuration.getEpollFacade(), + configuration.getWaitQueueCapacity() + ); + } catch (Throwable t) { + close(); + throw t; + } + } + + @Override + public void close() { + super.close(); + epoll = Misc.free(epoll); + } + + @Override + protected void ioWait(int timeout, int op) { + final int event = op == IOOperation.WRITE ? EpollAccessor.EPOLLOUT : EpollAccessor.EPOLLIN; + if (epoll.control(socket.getFd(), 0, EpollAccessor.EPOLL_CTL_MOD, event) < 0) { + throw new HttpClientException("internal error: epoll_ctl failure [op=").put(op) + .put(", errno=").put(nf.errno()) + .put(']'); + } + dieWaiting(epoll.poll(timeout)); + } + + @Override + protected void setupIoWait() { + if (epoll.control(socket.getFd(), 0, EpollAccessor.EPOLL_CTL_ADD, EpollAccessor.EPOLLOUT) < 0) { + throw new HttpClientException("internal error: epoll_ctl failure [cmd=add, errno=").put(nf.errno()).put(']'); + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientOsx.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientOsx.java new file mode 100644 index 0000000..b26b116 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientOsx.java @@ -0,0 +1,80 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.http.client; + +import io.questdb.client.HttpClientConfiguration; +import io.questdb.client.network.IOOperation; +import io.questdb.client.network.Kqueue; +import io.questdb.client.network.SocketFactory; +import io.questdb.client.std.Misc; + +/** + * macOS-specific WebSocket client using kqueue for I/O waiting. + */ +public class WebSocketClientOsx extends WebSocketClient { + private Kqueue kqueue; + + public WebSocketClientOsx(HttpClientConfiguration configuration, SocketFactory socketFactory) { + super(configuration, socketFactory); + try { + this.kqueue = new Kqueue( + configuration.getKQueueFacade(), + configuration.getWaitQueueCapacity() + ); + } catch (Throwable t) { + close(); + throw t; + } + } + + @Override + public void close() { + super.close(); + this.kqueue = Misc.free(kqueue); + } + + @Override + protected void ioWait(int timeout, int op) { + kqueue.setWriteOffset(0); + if (op == IOOperation.READ) { + kqueue.readFD(socket.getFd(), 0); + } else { + kqueue.writeFD(socket.getFd(), 0); + } + + // 1 = always one FD, we are a single threaded network client + if (kqueue.register(1) != 0) { + throw new HttpClientException("could not register with kqueue [op=").put(op) + .put(", errno=").errno(nf.errno()) + .put(']'); + } + dieWaiting(kqueue.poll(timeout)); + } + + @Override + protected void setupIoWait() { + // no-op on macOS + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientWindows.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientWindows.java new file mode 100644 index 0000000..3d00f82 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientWindows.java @@ -0,0 +1,79 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.http.client; + +import io.questdb.client.HttpClientConfiguration; +import io.questdb.client.network.FDSet; +import io.questdb.client.network.IOOperation; +import io.questdb.client.network.SelectFacade; +import io.questdb.client.network.SocketFactory; +import io.questdb.client.std.Misc; + +/** + * Windows-specific WebSocket client using select for I/O waiting. + */ +public class WebSocketClientWindows extends WebSocketClient { + private final SelectFacade sf; + private FDSet fdSet; + + public WebSocketClientWindows(HttpClientConfiguration configuration, SocketFactory socketFactory) { + super(configuration, socketFactory); + try { + this.fdSet = new FDSet(configuration.getWaitQueueCapacity()); + this.sf = configuration.getSelectFacade(); + } catch (Throwable t) { + close(); + throw t; + } + } + + @Override + public void close() { + super.close(); + this.fdSet = Misc.free(fdSet); + } + + @Override + protected void ioWait(int timeout, int op) { + final long readAddr; + final long writeAddr; + fdSet.clear(); + fdSet.add(socket.getFd()); + fdSet.setCount(1); + if (op == IOOperation.READ) { + readAddr = fdSet.address(); + writeAddr = 0; + } else { + readAddr = 0; + writeAddr = fdSet.address(); + } + dieWaiting(sf.select(readAddr, writeAddr, 0, timeout)); + } + + @Override + protected void setupIoWait() { + // no-op on Windows + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketFrameHandler.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketFrameHandler.java new file mode 100644 index 0000000..f0b8d8e --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketFrameHandler.java @@ -0,0 +1,93 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.http.client; + +/** + * Callback interface for handling received WebSocket frames. + *

+ * Implementations should process received data efficiently and avoid blocking, + * as callbacks are invoked on the I/O thread. + *

+ * Thread safety: Callbacks are invoked from the thread that called receiveFrame(). + * Implementations must handle their own synchronization if accessed from multiple threads. + */ +public interface WebSocketFrameHandler { + + /** + * Called when a binary frame is received. + * + * @param payloadPtr pointer to the payload data in native memory + * @param payloadLen length of the payload in bytes + */ + void onBinaryMessage(long payloadPtr, int payloadLen); + + /** + * Called when a close frame is received from the server. + *

+ * After this callback, the connection will be closed. The handler should + * perform any necessary cleanup. + * + * @param code the close status code (e.g., 1000 for normal closure) + * @param reason the close reason (may be null or empty) + */ + void onClose(int code, String reason); + + /** + * Called when a ping frame is received. + *

+ * Default implementation does nothing. The WebSocketClient automatically + * sends a pong response, so this callback is for informational purposes only. + * + * @param payloadPtr pointer to the ping payload in native memory + * @param payloadLen length of the payload in bytes + */ + default void onPing(long payloadPtr, int payloadLen) { + // Default: handled automatically by client + } + + /** + * Called when a pong frame is received. + *

+ * Default implementation does nothing. + * + * @param payloadPtr pointer to the pong payload in native memory + * @param payloadLen length of the payload in bytes + */ + default void onPong(long payloadPtr, int payloadLen) { + // Default: ignore pong frames + } + + /** + * Called when a text frame is received. + *

+ * Default implementation does nothing. Override if text frames need handling. + * + * @param payloadPtr pointer to the UTF-8 encoded payload in native memory + * @param payloadLen length of the payload in bytes + */ + default void onTextMessage(long payloadPtr, int payloadLen) { + // Default: ignore text frames + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java new file mode 100644 index 0000000..4b13481 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java @@ -0,0 +1,563 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.http.client; + +import io.questdb.client.cutlass.line.array.ArrayBufferAppender; +import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; +import io.questdb.client.cutlass.qwp.client.QwpBufferWriter; +import io.questdb.client.cutlass.qwp.websocket.WebSocketFrameWriter; +import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Numbers; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.SecureRnd; +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.Vect; + +/** + * Zero-GC WebSocket send buffer that implements {@link ArrayBufferAppender} for direct + * payload writing. Manages native memory with safe growth and handles WebSocket frame + * building (reserve header -> write payload -> patch header -> mask). + *

+ * Usage pattern: + * 

+ * buffer.beginBinaryFrame();
+ * // Write payload using ArrayBufferAppender methods
+ * buffer.putLong(value);
+ * buffer.putBlockOfBytes(ptr, len);
+ * // Finish frame and get send info
+ * FrameInfo frame = buffer.endBinaryFrame();
+ * // Send frame using socket
+ * socket.send(buffer.getBufferPtr() + frame.offset, frame.length);
+ * buffer.reset();
+ *
+ *

+ * Thread safety: This class is NOT thread-safe. Each connection should have its own buffer. + */ +public class WebSocketSendBuffer implements QwpBufferWriter, QuietCloseable { + + private static final int DEFAULT_INITIAL_CAPACITY = 65536; + private static final int MAX_BUFFER_SIZE = Integer.MAX_VALUE - 8; // Leave room for alignment + // Maximum header size: 2 (base) + 8 (64-bit length) + 4 (mask key) + private static final int MAX_HEADER_SIZE = 14; + private final FrameInfo frameInfo = new FrameInfo(); + private final int maxBufferSize; + private final SecureRnd rnd; + private int bufCapacity; + private long bufPtr; + private int frameStartOffset; // Where current frame's reserved header starts + private int payloadStartOffset; // Where payload begins (frameStart + MAX_HEADER_SIZE) + private int writePos; // Current write position (offset from bufPtr) + + /** + * Creates a new WebSocket send buffer with default initial capacity. + */ + public WebSocketSendBuffer() { + this(DEFAULT_INITIAL_CAPACITY, MAX_BUFFER_SIZE); + } + + /** + * Creates a new WebSocket send buffer with specified initial capacity. + * + * @param initialCapacity initial buffer size in bytes + */ + public WebSocketSendBuffer(int initialCapacity) { + this(initialCapacity, MAX_BUFFER_SIZE); + } + + /** + * Creates a new WebSocket send buffer with specified initial and max capacity. + * + * @param initialCapacity initial buffer size in bytes + * @param maxBufferSize maximum buffer size in bytes + */ + public WebSocketSendBuffer(int initialCapacity, int maxBufferSize) { + this.bufCapacity = Math.max(initialCapacity, MAX_HEADER_SIZE * 2); + this.maxBufferSize = maxBufferSize; + this.bufPtr = Unsafe.malloc(bufCapacity, MemoryTag.NATIVE_DEFAULT); + this.writePos = 0; + this.frameStartOffset = 0; + this.payloadStartOffset = 0; + this.rnd = new SecureRnd(); + } + + /** + * Begins a new WebSocket frame. Reserves space for the maximum header size. + * The opcode is specified later when ending the frame via {@link #endFrame(int)}. + */ + public void beginFrame() { + frameStartOffset = writePos; + // Reserve maximum header space + ensureCapacity(MAX_HEADER_SIZE); + writePos += MAX_HEADER_SIZE; + payloadStartOffset = writePos; + } + + @Override + public void close() { + if (bufPtr != 0) { + Unsafe.free(bufPtr, bufCapacity, MemoryTag.NATIVE_DEFAULT); + bufPtr = 0; + bufCapacity = 0; + } + } + + /** + * Finishes the current binary frame, writing the header and applying masking. + * Returns information about where to find the complete frame in the buffer. + *

+ * IMPORTANT: Only call this after all payload writes are complete. The buffer + * pointer is stable after this call (no more reallocations for this frame). + * + * @return frame info containing offset and length for sending + */ + public FrameInfo endBinaryFrame() { + return endFrame(WebSocketOpcode.BINARY); + } + + /** + * Finishes the current frame with the specified opcode. + * + * @param opcode the frame opcode + * @return frame info containing offset and length for sending + */ + public FrameInfo endFrame(int opcode) { + int payloadLen = writePos - payloadStartOffset; + + // Calculate actual header size (with mask key for client frames) + int actualHeaderSize = WebSocketFrameWriter.headerSize(payloadLen, true); + int unusedSpace = MAX_HEADER_SIZE - actualHeaderSize; + int actualFrameStart = frameStartOffset + unusedSpace; + + // Generate mask key + int maskKey = rnd.nextInt(); + + // Write header at actual position (after unused space) + WebSocketFrameWriter.writeHeader(bufPtr + actualFrameStart, true, opcode, payloadLen, maskKey); + + // Apply mask to payload + if (payloadLen > 0) { + WebSocketFrameWriter.maskPayload(bufPtr + payloadStartOffset, payloadLen, maskKey); + } + + return frameInfo.set(actualFrameStart, actualHeaderSize + payloadLen); + } + + /** + * Finishes the current text frame, writing the header and applying masking. + */ + public FrameInfo endTextFrame() { + return endFrame(WebSocketOpcode.TEXT); + } + + /** + * Ensures the buffer has capacity for the specified number of additional bytes. + * May reallocate the buffer if necessary. + * + * @param additionalBytes number of additional bytes needed + */ + @Override + public void ensureCapacity(int additionalBytes) { + long requiredCapacity = (long) writePos + additionalBytes; + if (requiredCapacity > bufCapacity) { + grow(requiredCapacity); + } + } + + /** + * Gets the buffer pointer. Only use this for reading after frame is complete. + */ + public long getBufferPtr() { + return bufPtr; + } + + /** + * Gets the current buffer capacity. + */ + public int getCapacity() { + return bufCapacity; + } + + /** + * Gets the payload length of the current frame being built. + */ + public int getCurrentPayloadLength() { + return writePos - payloadStartOffset; + } + + /** + * Gets the current write position (number of bytes written). + */ + @Override + public int getPosition() { + return writePos; + } + + /** + * Gets the current write position (total bytes written since last reset). + */ + public int getWritePos() { + return writePos; + } + + /** + * Patches an int value at the specified offset. + */ + @Override + public void patchInt(int offset, int value) { + Unsafe.getUnsafe().putInt(bufPtr + offset, value); + } + + /** + * Writes an ASCII string. + */ + public void putAscii(CharSequence cs) { + if (cs == null) { + return; + } + int len = cs.length(); + ensureCapacity(len); + for (int i = 0; i < len; i++) { + Unsafe.getUnsafe().putByte(bufPtr + writePos + i, (byte) cs.charAt(i)); + } + writePos += len; + } + + @Override + public void putBlockOfBytes(long from, long len) { + if (len <= 0) { + return; + } + if (len > Integer.MAX_VALUE) { + throw new IllegalArgumentException("len exceeds int range: " + len); + } + int intLen = (int) len; + ensureCapacity(intLen); + Vect.memcpy(bufPtr + writePos, from, intLen); + writePos += intLen; + } + + @Override + public void putByte(byte b) { + ensureCapacity(1); + Unsafe.getUnsafe().putByte(bufPtr + writePos, b); + writePos++; + } + + /** + * Writes raw bytes from a byte array. + */ + public void putBytes(byte[] bytes, int offset, int length) { + if (length <= 0) { + return; + } + ensureCapacity(length); + for (int i = 0; i < length; i++) { + Unsafe.getUnsafe().putByte(bufPtr + writePos + i, bytes[offset + i]); + } + writePos += length; + } + + @Override + public void putDouble(double value) { + ensureCapacity(8); + Unsafe.getUnsafe().putDouble(bufPtr + writePos, value); + writePos += 8; + } + + /** + * Writes a float value. + */ + public void putFloat(float value) { + ensureCapacity(4); + Unsafe.getUnsafe().putFloat(bufPtr + writePos, value); + writePos += 4; + } + + @Override + public void putInt(int value) { + ensureCapacity(4); + Unsafe.getUnsafe().putInt(bufPtr + writePos, value); + writePos += 4; + } + + @Override + public void putLong(long value) { + ensureCapacity(8); + Unsafe.getUnsafe().putLong(bufPtr + writePos, value); + writePos += 8; + } + + /** + * Writes a long value in big-endian format. + */ + public void putLongBE(long value) { + ensureCapacity(8); + Unsafe.getUnsafe().putLong(bufPtr + writePos, Long.reverseBytes(value)); + writePos += 8; + } + + /** + * Writes a short value in little-endian format. + */ + public void putShort(short value) { + ensureCapacity(2); + Unsafe.getUnsafe().putShort(bufPtr + writePos, value); + writePos += 2; + } + + /** + * Writes a length-prefixed UTF-8 string. + */ + @Override + public void putString(String value) { + if (value == null || value.isEmpty()) { + putVarint(0); + return; + } + int utf8Len = NativeBufferWriter.utf8Length(value); + putVarint(utf8Len); + putUtf8(value); + } + + /** + * Writes UTF-8 encoded bytes directly without length prefix. + */ + @Override + public void putUtf8(String value) { + if (value == null || value.isEmpty()) { + return; + } + for (int i = 0, n = value.length(); i < n; i++) { + char c = value.charAt(i); + if (c < 0x80) { + putByte((byte) c); + } else if (c < 0x800) { + putByte((byte) (0xC0 | (c >> 6))); + putByte((byte) (0x80 | (c & 0x3F))); + } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n) { + char c2 = value.charAt(++i); + if (Character.isLowSurrogate(c2)) { + int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); + putByte((byte) (0xF0 | (codePoint >> 18))); + putByte((byte) (0x80 | ((codePoint >> 12) & 0x3F))); + putByte((byte) (0x80 | ((codePoint >> 6) & 0x3F))); + putByte((byte) (0x80 | (codePoint & 0x3F))); + } else { + putByte((byte) '?'); + i--; + } + } else if (Character.isSurrogate(c)) { + putByte((byte) '?'); + } else { + putByte((byte) (0xE0 | (c >> 12))); + putByte((byte) (0x80 | ((c >> 6) & 0x3F))); + putByte((byte) (0x80 | (c & 0x3F))); + } + } + } + + /** + * Writes an unsigned variable-length integer (LEB128 encoding). + */ + @Override + public void putVarint(long value) { + while (value > 0x7F) { + putByte((byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + putByte((byte) value); + } + + /** + * Resets the buffer for reuse. Does not deallocate memory. + */ + public void reset() { + writePos = 0; + frameStartOffset = 0; + payloadStartOffset = 0; + } + + /** + * Skips the specified number of bytes, advancing the position. + */ + @Override + public void skip(int bytes) { + ensureCapacity(bytes); + writePos += bytes; + } + + /** + * Writes a complete close frame. + * + * @param code close status code (e.g., 1000 for normal closure) + * @param reason optional reason string (may be null) + * @return frame info for sending + */ + public FrameInfo writeCloseFrame(int code, String reason) { + int payloadLen = 2; // status code + byte[] reasonBytes = null; + if (reason != null && !reason.isEmpty()) { + reasonBytes = reason.getBytes(java.nio.charset.StandardCharsets.UTF_8); + payloadLen += reasonBytes.length; + } + + if (payloadLen > 125) { + throw new HttpClientException("Close payload too large [len=").put(payloadLen).put(']'); + } + + int frameStart = writePos; + int headerSize = WebSocketFrameWriter.headerSize(payloadLen, true); + ensureCapacity(headerSize + payloadLen); + + int maskKey = rnd.nextInt(); + int written = WebSocketFrameWriter.writeHeader(bufPtr + writePos, true, WebSocketOpcode.CLOSE, payloadLen, maskKey); + writePos += written; + + // Write status code (big-endian) + long payloadStart = bufPtr + writePos; + Unsafe.getUnsafe().putByte(payloadStart, (byte) ((code >> 8) & 0xFF)); + Unsafe.getUnsafe().putByte(payloadStart + 1, (byte) (code & 0xFF)); + writePos += 2; + + // Write reason if present + if (reasonBytes != null) { + for (byte reasonByte : reasonBytes) { + Unsafe.getUnsafe().putByte(bufPtr + writePos++, reasonByte); + } + } + + // Mask the payload (including status code and reason) + WebSocketFrameWriter.maskPayload(payloadStart, payloadLen, maskKey); + + return frameInfo.set(frameStart, headerSize + payloadLen); + } + + /** + * Writes a complete ping frame (control frame, no masking needed for server). + * Note: Client frames MUST be masked per RFC 6455. This writes a masked ping. + * + * @return frame info for sending + */ + public FrameInfo writePingFrame() { + return writePingFrame(0, 0); + } + + /** + * Writes a complete ping frame with payload. + * + * @param payloadPtr pointer to ping payload + * @param payloadLen length of payload (max 125 bytes for control frames) + * @return frame info for sending + */ + public FrameInfo writePingFrame(long payloadPtr, int payloadLen) { + if (payloadLen > 125) { + throw new HttpClientException("Ping payload too large [len=").put(payloadLen).put(']'); + } + + int frameStart = writePos; + int headerSize = WebSocketFrameWriter.headerSize(payloadLen, true); + ensureCapacity(headerSize + payloadLen); + + int maskKey = rnd.nextInt(); + int written = WebSocketFrameWriter.writeHeader(bufPtr + writePos, true, WebSocketOpcode.PING, payloadLen, maskKey); + writePos += written; + + if (payloadLen > 0) { + Vect.memcpy(bufPtr + writePos, payloadPtr, payloadLen); + WebSocketFrameWriter.maskPayload(bufPtr + writePos, payloadLen, maskKey); + writePos += payloadLen; + } + + return frameInfo.set(frameStart, headerSize + payloadLen); + } + + /** + * Writes a complete pong frame. + * + * @param payloadPtr pointer to pong payload (should match received ping) + * @param payloadLen length of payload + * @return frame info for sending + */ + public FrameInfo writePongFrame(long payloadPtr, int payloadLen) { + if (payloadLen > 125) { + throw new HttpClientException("Pong payload too large [len=").put(payloadLen).put(']'); + } + + int frameStart = writePos; + int headerSize = WebSocketFrameWriter.headerSize(payloadLen, true); + ensureCapacity(headerSize + payloadLen); + + int maskKey = rnd.nextInt(); + int written = WebSocketFrameWriter.writeHeader(bufPtr + writePos, true, WebSocketOpcode.PONG, payloadLen, maskKey); + writePos += written; + + if (payloadLen > 0) { + Vect.memcpy(bufPtr + writePos, payloadPtr, payloadLen); + WebSocketFrameWriter.maskPayload(bufPtr + writePos, payloadLen, maskKey); + writePos += payloadLen; + } + + return frameInfo.set(frameStart, headerSize + payloadLen); + } + + private void grow(long requiredCapacity) { + if (requiredCapacity > maxBufferSize) { + throw new HttpClientException("WebSocket buffer size exceeded maximum [required=") + .put(requiredCapacity) + .put(", max=") + .put(maxBufferSize) + .put(']'); + } + int newCapacity = Math.min( + Math.max(Numbers.ceilPow2((int) requiredCapacity), (int) requiredCapacity), + maxBufferSize + ); + bufPtr = Unsafe.realloc(bufPtr, bufCapacity, newCapacity, MemoryTag.NATIVE_DEFAULT); + bufCapacity = newCapacity; + } + + /** + * Information about a completed WebSocket frame's location in the buffer. + * This class is mutable and reused to avoid allocations. Callers must + * extract values before calling any end*Frame() method again. + */ + public static final class FrameInfo { + /** + * Total length of the frame (header + payload). + */ + public int length; + /** + * Offset from buffer start where the frame begins. + */ + public int offset; + + FrameInfo set(int offset, int length) { + this.offset = offset; + this.length = length; + return this; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/json/JsonLexer.java b/core/src/main/java/io/questdb/client/cutlass/json/JsonLexer.java index e8aa4d0..565a023 100644 --- a/core/src/main/java/io/questdb/client/cutlass/json/JsonLexer.java +++ b/core/src/main/java/io/questdb/client/cutlass/json/JsonLexer.java @@ -65,6 +65,7 @@ public class JsonLexer implements Mutable, Closeable { private boolean quoted = false; private int state = S_START; private boolean useCache = false; + public JsonLexer(int cacheSize, int cacheSizeLimit) { this.cacheSizeLimit = cacheSizeLimit; // if cacheSizeLimit is 0 or negative, the cache is disabled @@ -398,4 +399,4 @@ private void utf8DecodeCacheAndBuffer(long lo, long hi, int position) throws Jso unquotedTerminators.add('{'); unquotedTerminators.add('['); } -} \ No newline at end of file +} diff --git a/core/src/main/java/io/questdb/client/cutlass/line/AbstractLineSender.java b/core/src/main/java/io/questdb/client/cutlass/line/AbstractLineSender.java index 9a0d06b..1c3953b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/line/AbstractLineSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/line/AbstractLineSender.java @@ -368,7 +368,7 @@ private static int findEOL(long ptr, int len) { private byte[] receiveChallengeBytes() { int n = 0; - for (;;) { + for (; ; ) { int rc = lineChannel.receive(ptr + n, capacity - n); if (rc < 0) { int errno = lineChannel.errno(); @@ -505,4 +505,4 @@ protected AbstractLineSender writeFieldName(CharSequence name) { } throw new LineSenderException("table expected"); } -} \ No newline at end of file +} diff --git a/core/src/main/java/io/questdb/client/cutlass/line/LineSenderException.java b/core/src/main/java/io/questdb/client/cutlass/line/LineSenderException.java index 74af2f6..b599efc 100644 --- a/core/src/main/java/io/questdb/client/cutlass/line/LineSenderException.java +++ b/core/src/main/java/io/questdb/client/cutlass/line/LineSenderException.java @@ -39,15 +39,9 @@ *

  • For permanent errors: Either close and recreate the Sender, or call {@code reset()} to clear * the buffer and continue with new data
    • * + *

    + * @see io.questdb.client.Sender * - *

    Retryability

    - * The {@link #isRetryable()} method provides a best-effort indication of whether the error - * might be resolved by retrying at the application level. This is particularly important - * because this exception is only thrown after the sender has exhausted its own internal - * retry attempts. The retryability flag helps applications decide whether to implement - * additional retry logic with longer delays or different strategies. - * - * @see io.questdb.client.Sender * @see io.questdb.client.Sender#flush() * @see io.questdb.client.Sender#reset() */ @@ -115,4 +109,4 @@ public LineSenderException putAsPrintable(CharSequence nonPrintable) { message.putAsPrintable(nonPrintable); return this; } -} \ No newline at end of file +} diff --git a/core/src/main/java/io/questdb/client/cutlass/line/udp/UdpLineChannel.java b/core/src/main/java/io/questdb/client/cutlass/line/udp/UdpLineChannel.java index e630bcf..e89a261 100644 --- a/core/src/main/java/io/questdb/client/cutlass/line/udp/UdpLineChannel.java +++ b/core/src/main/java/io/questdb/client/cutlass/line/udp/UdpLineChannel.java @@ -85,4 +85,10 @@ public void send(long ptr, int len) { throw new LineSenderException("send error").errno(nf.errno()); } } + + public void sendSegments(long segmentsPtr, int segmentCount, int totalLen) { + if (nf.sendToRawScatter(fd, segmentsPtr, segmentCount, sockaddr) != totalLen) { + throw new LineSenderException("send error").errno(nf.errno()); + } + } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java new file mode 100644 index 0000000..3b4c9a9 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java @@ -0,0 +1,146 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.std.CharSequenceIntHashMap; +import io.questdb.client.std.ObjList; + +/** + * Global symbol dictionary that maps symbol strings to sequential integer IDs. + *

    + * This dictionary is shared across all tables and columns within a client instance. + * IDs are assigned sequentially starting from 0, ensuring contiguous ID space. + *

    + * Thread safety: This class is NOT thread-safe. External synchronization is required + * if accessed from multiple threads. + */ +public class GlobalSymbolDictionary { + + private final ObjList idToSymbol; + private final CharSequenceIntHashMap symbolToId; + + public GlobalSymbolDictionary() { + this(64); // Default initial capacity + } + + public GlobalSymbolDictionary(int initialCapacity) { + this.symbolToId = new CharSequenceIntHashMap(initialCapacity); + this.idToSymbol = new ObjList<>(initialCapacity); + } + + /** + * Clears all symbols from the dictionary. + *

    + * After clearing, the next symbol added will get ID 0. + */ + public void clear() { + symbolToId.clear(); + idToSymbol.clear(); + } + + /** + * Checks if the dictionary contains the given symbol. + * + * @param symbol the symbol to check + * @return true if the symbol exists in the dictionary + */ + public boolean contains(String symbol) { + return symbol != null && symbolToId.get(symbol) != CharSequenceIntHashMap.NO_ENTRY_VALUE; + } + + /** + * Gets the ID for an existing symbol, or -1 if not found. + * + * @param symbol the symbol string + * @return the symbol ID, or -1 if not in dictionary + */ + public int getId(String symbol) { + if (symbol == null) { + return -1; + } + int id = symbolToId.get(symbol); + return id == CharSequenceIntHashMap.NO_ENTRY_VALUE ? -1 : id; + } + + /** + * Gets or adds a symbol to the dictionary. + *

    + * If the symbol already exists, returns its existing ID. + * If the symbol is new, assigns the next sequential ID and returns it. + * + * @param symbol the symbol string (must not be null) + * @return the global ID for this symbol (>= 0) + * @throws IllegalArgumentException if symbol is null + */ + public int getOrAddSymbol(CharSequence symbol) { + if (symbol == null) { + throw new IllegalArgumentException("symbol cannot be null"); + } + + int existingId = symbolToId.get(symbol); + if (existingId != CharSequenceIntHashMap.NO_ENTRY_VALUE) { + return existingId; + } + + // Assign new ID — toString() only for new symbols that must be stored + String symbolStr = symbol.toString(); + int newId = idToSymbol.size(); + symbolToId.put(symbolStr, newId); + idToSymbol.add(symbolStr); + return newId; + } + + /** + * Gets the symbol string for a given ID. + * + * @param id the symbol ID + * @return the symbol string + * @throws IndexOutOfBoundsException if id is out of range + */ + public String getSymbol(int id) { + if (id < 0 || id >= idToSymbol.size()) { + throw new IndexOutOfBoundsException("Invalid symbol ID: " + id + ", dictionary size: " + idToSymbol.size()); + } + return idToSymbol.getQuick(id); + } + + /** + * Checks if the dictionary is empty. + * + * @return true if no symbols have been added + */ + public boolean isEmpty() { + return idToSymbol.size() == 0; + } + + /** + * Returns the number of symbols in the dictionary. + * + * @return dictionary size + */ + public int size() { + return idToSymbol.size(); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java new file mode 100644 index 0000000..da56a1c --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java @@ -0,0 +1,484 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.cutlass.line.LineSenderException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.LockSupport; + +/** + * Lock-free in-flight batch tracker for the sliding window protocol. + *

    + * Concurrency model (lock-free): + *

      + *
    • Async mode: the WebSocket I/O thread sends and receives; it calls + * {@link #tryAddInFlight(long)} before send and {@link #acknowledgeUpTo(long)} + * on ACKs (single writer for sent and acked).
      • + *
    • Sync mode: the caller thread sends and waits synchronously; it calls + * {@link #addInFlight(long)} (window size = 1) then waits for ACK itself on + * the same thread, so the window is always drained inline.
      • + *
    • Waiter: in async mode the caller thread may call {@link #awaitEmpty()} + * during flush to wait for the window to drain; it only reads the counters and + * parks/unparks.
      • + *
    + * Assumptions that keep it simple and lock-free: + *
      + *
    • Batch IDs are sequential (sender increments by 1)
      • + *
    • Single producer updates {@code highestSent}
      • + *
    • Single consumer updates {@code highestAcked}
      • + *
    + * With these constraints we can rely on volatile reads/writes (no CAS) and still + * offer blocking waits for space/empty without protecting the counters with locks. + */ +public class InFlightWindow { + + public static final long DEFAULT_TIMEOUT_MS = 30_000; + public static final int DEFAULT_WINDOW_SIZE = 8; + private static final Logger LOG = LoggerFactory.getLogger(InFlightWindow.class); + private static final long PARK_NANOS = 100_000; // 100 microseconds + // Spin parameters + private static final int SPIN_TRIES = 100; + private static final VarHandle TOTAL_ACKED; + private static final VarHandle TOTAL_FAILED; + + static { + try { + MethodHandles.Lookup lookup = MethodHandles.lookup(); + TOTAL_ACKED = lookup.findVarHandle(InFlightWindow.class, "totalAcked", long.class); + TOTAL_FAILED = lookup.findVarHandle(InFlightWindow.class, "totalFailed", long.class); + } catch (ReflectiveOperationException e) { + throw new ExceptionInInitializerError(e); + } + } + // Error state + private final AtomicReference lastError = new AtomicReference<>(); + private final int maxWindowSize; + private final long timeoutMs; + private volatile long failedBatchId = -1; + // highestAcked: the sequence number of the last acknowledged batch (cumulative) + private volatile long highestAcked = -1; + // Core state + // highestSent: the sequence number of the last batch added to the window + private volatile long highestSent = -1; + // Statistics — updated atomically via VarHandle + private long totalAcked = 0; + private long totalFailed = 0; + // Thread waiting for empty (flush thread) + private volatile Thread waitingForEmpty; + // Thread waiting for space (sender thread) + private volatile Thread waitingForSpace; + + /** + * Creates a new InFlightWindow with default configuration. + */ + public InFlightWindow() { + this(DEFAULT_WINDOW_SIZE, DEFAULT_TIMEOUT_MS); + } + + /** + * Creates a new InFlightWindow with custom configuration. + * + * @param maxWindowSize maximum number of batches in flight + * @param timeoutMs timeout for blocking operations + */ + public InFlightWindow(int maxWindowSize, long timeoutMs) { + if (maxWindowSize <= 0) { + throw new IllegalArgumentException("maxWindowSize must be positive"); + } + this.maxWindowSize = maxWindowSize; + this.timeoutMs = timeoutMs; + } + + /** + * Acknowledges a batch, removing it from the in-flight window. + *

    + * For sequential batch IDs, this is a cumulative acknowledgment - + * acknowledging batch N means all batches up to N are acknowledged. + *

    + * Called by: acker (WebSocket I/O thread) after receiving an ACK. + * + * @param batchId the batch ID that was acknowledged + * @return true if the batch was in flight, false if already acknowledged + */ + public boolean acknowledge(long batchId) { + return acknowledgeUpTo(batchId) > 0 || highestAcked >= batchId; + } + + /** + * Acknowledges all batches up to and including the given sequence (cumulative ACK). + * Lock-free with single consumer. + *

    + * Called by: acker (WebSocket I/O thread) after receiving an ACK. + * + * @param sequence the highest acknowledged sequence + * @return the number of batches acknowledged + */ + public int acknowledgeUpTo(long sequence) { + long sent = highestSent; + + // Nothing to acknowledge if window is empty or sequence is beyond what's sent + if (sent < 0) { + return 0; // No batches have been sent + } + + // Cap sequence at highestSent - can't acknowledge what hasn't been sent + long effectiveSequence = Math.min(sequence, sent); + + long prevAcked = highestAcked; + if (effectiveSequence <= prevAcked) { + // Already acknowledged up to this point + return 0; + } + highestAcked = effectiveSequence; + + int acknowledged = (int) (effectiveSequence - prevAcked); + TOTAL_ACKED.getAndAdd(this, (long) acknowledged); + + LOG.debug("Cumulative ACK [upTo={}, acknowledged={}, remaining={}]", sequence, acknowledged, getInFlightCount()); + + // Wake up waiting threads + Thread waiter = waitingForSpace; + if (waiter != null) { + LockSupport.unpark(waiter); + } + + waiter = waitingForEmpty; + if (waiter != null && getInFlightCount() == 0) { + LockSupport.unpark(waiter); + } + + return acknowledged; + } + + /** + * Adds a batch to the in-flight window. + *

    + * Blocks if the window is full until space becomes available or timeout. + * Uses spin-wait with exponential backoff, then parks. Blocking is only expected + * in modes where another actor can make progress on acknowledgments. In normal + * sync usage the window size is 1 and the same thread immediately waits for the + * ACK, so this should never actually park. If a caller uses a larger window here + * it must ensure ACKs are processed on another thread; a single-threaded caller + * with window>1 would deadlock by parking while also being the only thread that + * can advance {@link #acknowledgeUpTo(long)}. + *

    + * Called by: sync sender thread before sending a batch (window=1). + * + * @param batchId the batch ID to track + * @throws LineSenderException if timeout occurs or an error was reported + */ + public void addInFlight(long batchId) { + // Check for errors first + checkError(); + + // Fast path: try to add without waiting + if (tryAddInFlightInternal(batchId)) { + return; + } + + // Slow path: need to wait for space + long deadline = System.currentTimeMillis() + timeoutMs; + int spins = 0; + + // Register as waiting thread + waitingForSpace = Thread.currentThread(); + try { + while (true) { + // Check for errors + checkError(); + + // Try to add + if (tryAddInFlightInternal(batchId)) { + return; + } + + // Check timeout + long remaining = deadline - System.currentTimeMillis(); + if (remaining <= 0) { + throw new LineSenderException("Timeout waiting for window space, window full with " + + getInFlightCount() + " batches"); + } + + // Spin or park + if (spins < SPIN_TRIES) { + Thread.onSpinWait(); + spins++; + } else { + // Park with timeout + LockSupport.parkNanos(Math.min(PARK_NANOS, remaining * 1_000_000)); + if (Thread.interrupted()) { + throw new LineSenderException("Interrupted while waiting for window space"); + } + } + } + } finally { + waitingForSpace = null; + } + } + + /** + * Waits until all in-flight batches are acknowledged. + *

    + * Called by flush() to ensure all data is confirmed. + *

    + * Called by: waiter (flush thread), while producer/acker thread progresses. + * + * @throws LineSenderException if timeout occurs or an error was reported + */ + public void awaitEmpty() { + checkError(); + + // Fast path: already empty + if (getInFlightCount() == 0) { + LOG.debug("Window already empty"); + return; + } + + long deadline = System.currentTimeMillis() + timeoutMs; + int spins = 0; + + // Register as waiting thread + waitingForEmpty = Thread.currentThread(); + try { + while (getInFlightCount() > 0) { + checkError(); + + long remaining = deadline - System.currentTimeMillis(); + if (remaining <= 0) { + throw new LineSenderException("Timeout waiting for batch acknowledgments, " + + getInFlightCount() + " batches still in flight"); + } + + if (spins < SPIN_TRIES) { + Thread.onSpinWait(); + spins++; + } else { + LockSupport.parkNanos(Math.min(PARK_NANOS, remaining * 1_000_000)); + if (Thread.interrupted()) { + throw new LineSenderException("Interrupted while waiting for acknowledgments"); + } + } + } + + // The I/O thread may have called fail() and then acknowledgeUpTo() + // before this thread was scheduled, draining the window while an + // error is pending. Check one final time after the window is empty. + checkError(); + + LOG.debug("Window empty, all batches ACKed"); + } finally { + waitingForEmpty = null; + } + } + + /** + * Clears the error state. + */ + public void clearError() { + lastError.set(null); + failedBatchId = -1; + } + + /** + * Marks a batch as failed, setting an error that will be propagated to waiters. + *

    + * Called by: acker (WebSocket I/O thread) on error response or send failure. + * + * @param batchId the batch ID that failed + * @param error the error that occurred + */ + public void fail(long batchId, Throwable error) { + this.failedBatchId = batchId; + this.lastError.set(error); + TOTAL_FAILED.getAndAdd(this, 1L); + + LOG.error("Batch failed [batchId={}, error={}]", batchId, String.valueOf(error)); + + wakeWaiters(); + } + + /** + * Marks all currently in-flight batches as failed. + *

    + * Used for transport-level failures (disconnect/protocol violation) where + * no further ACKs are expected and all waiters must be released. + * + * @param error terminal error to propagate + */ + public void failAll(Throwable error) { + long sent = highestSent; + long acked = highestAcked; + long inFlight = Math.max(0, sent - acked); + + this.failedBatchId = sent; + this.lastError.set(error); + TOTAL_FAILED.getAndAdd(this, Math.max(1L, inFlight)); + + LOG.error("All in-flight batches failed [inFlight={}, error={}]", inFlight, String.valueOf(error)); + + wakeWaiters(); + } + + /** + * Returns the current number of batches in flight. + * Wait-free operation. + */ + public int getInFlightCount() { + long sent = highestSent; + long acked = highestAcked; + // Ensure non-negative (can happen during initialization) + return (int) Math.max(0, sent - acked); + } + + /** + * Returns the last error, or null if no error. + */ + public Throwable getLastError() { + return lastError.get(); + } + + /** + * Returns the maximum window size. + */ + public int getMaxWindowSize() { + return maxWindowSize; + } + + /** + * Returns the total number of batches acknowledged. + */ + public long getTotalAcked() { + return (long) TOTAL_ACKED.getOpaque(this); + } + + /** + * Returns the total number of batches that failed. + */ + public long getTotalFailed() { + return (long) TOTAL_FAILED.getOpaque(this); + } + + /** + * Checks if there's space in the window for another batch. + * Wait-free operation. + * + * @return true if there's space, false if window is full + */ + public boolean hasWindowSpace() { + return getInFlightCount() < maxWindowSize; + } + + /** + * Returns true if the window is empty. + * Wait-free operation. + */ + public boolean isEmpty() { + return getInFlightCount() == 0; + } + + /** + * Returns true if the window is full. + * Wait-free operation. + */ + public boolean isFull() { + return getInFlightCount() >= maxWindowSize; + } + + /** + * Resets the window, clearing all state. + */ + public void reset() { + highestSent = -1; + highestAcked = -1; + lastError.set(null); + failedBatchId = -1; + + wakeWaiters(); + } + + /** + * Tries to add a batch to the in-flight window without blocking. + * Lock-free, assuming single producer for highestSent. + *

    + * Called by: async producer (WebSocket I/O thread) before sending a batch. + * + * @param batchId the batch ID to track (must be sequential) + * @return true if added, false if window is full + */ + public boolean tryAddInFlight(long batchId) { + // Check window space first + long sent = highestSent; + long acked = highestAcked; + + if (sent - acked >= maxWindowSize) { + return false; + } + + // Sequential caller: just publish the new highestSent + highestSent = batchId; + + LOG.debug("Added to window [batchId={}, windowSize={}]", batchId, getInFlightCount()); + return true; + } + + private void checkError() { + Throwable error = lastError.get(); + if (error != null) { + throw new LineSenderException("Batch " + failedBatchId + " failed: " + error.getMessage(), error); + } + } + + private boolean tryAddInFlightInternal(long batchId) { + long sent = highestSent; + long acked = highestAcked; + + if (sent - acked >= maxWindowSize) { + return false; + } + + // For sequential IDs, we just update highestSent + // The caller guarantees batchId is the next in sequence + highestSent = batchId; + + LOG.debug("Added to window [batchId={}, windowSize={}]", batchId, getInFlightCount()); + return true; + } + + private void wakeWaiters() { + Thread waiter = waitingForSpace; + if (waiter != null) { + LockSupport.unpark(waiter); + } + waiter = waitingForEmpty; + if (waiter != null) { + LockSupport.unpark(waiter); + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java new file mode 100644 index 0000000..aa19541 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java @@ -0,0 +1,454 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.LockSupport; + +/** + * A buffer for accumulating ILP data into microbatches before sending. + *

    + * This class implements a state machine for buffer lifecycle management in the + * double-buffering scheme used by {@link QwpWebSocketSender}: + * 

    + * Buffer States:
+ * ┌─────────────┐    seal()     ┌─────────────┐    markSending()  ┌─────────────┐
+ * │   FILLING   │──────────────►│   SEALED    │──────────────────►│   SENDING   │
+ * │ (user owns) │               │ (in queue)  │                   │ (I/O owns)  │
+ * └─────────────┘               └─────────────┘                   └──────┬──────┘
+ *        ▲                                                               │
+ *        │                         markRecycled()                        │
+ *        └───────────────────────────────────────────────────────────────┘
+ *                              (after send complete)
+ *
    + *

    + * Thread safety: This class is NOT thread-safe for concurrent writes. However, it + * supports safe hand-over between user thread and I/O thread through the state + * machine. State transitions use volatile fields to ensure visibility. + */ +public class MicrobatchBuffer implements QuietCloseable { + + // Buffer states + public static final int STATE_FILLING = 0; + public static final int STATE_RECYCLED = 3; + public static final int STATE_SEALED = 1; + public static final int STATE_SENDING = 2; + private static final AtomicLong nextBatchId = new AtomicLong(); + // Flush trigger thresholds + // Batch identification + private long batchId; + private int bufferCapacity; + private int bufferPos; + // Native memory buffer + private long bufferPtr; + private long firstRowTimeNanos; + // For waiting on recycle (user thread waits for I/O thread to finish) + private volatile Thread recycleWaiter; + // Row tracking + private int rowCount; + // State machine + private volatile int state = STATE_FILLING; + + /** + * Creates a new MicrobatchBuffer with specified flush thresholds. + * + * @param initialCapacity initial buffer size in bytes + * @param maxRows maximum rows before auto-flush (0 = unlimited) + * @param maxBytes maximum bytes before auto-flush (0 = unlimited) + * @param maxAgeNanos maximum age in nanoseconds before auto-flush (0 = unlimited) + */ + public MicrobatchBuffer(int initialCapacity, int maxRows, int maxBytes, long maxAgeNanos) { + if (initialCapacity <= 0) { + throw new IllegalArgumentException("initialCapacity must be positive"); + } + this.bufferCapacity = initialCapacity; + this.bufferPtr = Unsafe.malloc(initialCapacity, MemoryTag.NATIVE_ILP_RSS); + this.bufferPos = 0; + this.rowCount = 0; + this.firstRowTimeNanos = 0; + this.batchId = nextBatchId.getAndIncrement(); + } + + /** + * Creates a new MicrobatchBuffer with default thresholds (no auto-flush). + * + * @param initialCapacity initial buffer size in bytes + */ + public MicrobatchBuffer(int initialCapacity) { + this(initialCapacity, 0, 0, 0); + } + + /** + * Returns a human-readable name for the given state. + */ + public static String stateName(int state) { + switch (state) { + case STATE_FILLING: + return "FILLING"; + case STATE_SEALED: + return "SEALED"; + case STATE_SENDING: + return "SENDING"; + case STATE_RECYCLED: + return "RECYCLED"; + default: + return "UNKNOWN(" + state + ")"; + } + } + + /** + * Waits for the buffer to be recycled (transition to RECYCLED state). + * Only the user thread should call this. + */ + public void awaitRecycled() { + final Thread current = Thread.currentThread(); + recycleWaiter = current; + try { + while (state != STATE_RECYCLED) { + LockSupport.park(this); + if (Thread.interrupted()) { + Thread.currentThread().interrupt(); + return; + } + } + } finally { + if (recycleWaiter == current) { + recycleWaiter = null; + } + } + } + + /** + * Waits for the buffer to be recycled with a timeout. + * + * @param timeout the maximum time to wait + * @param unit the time unit + * @return true if recycled, false if timeout elapsed + */ + public boolean awaitRecycled(long timeout, TimeUnit unit) { + if (state == STATE_RECYCLED) { + // fast-path + return true; + } + + final Thread current = Thread.currentThread(); + recycleWaiter = current; + final long deadlineNanos = System.nanoTime() + unit.toNanos(timeout); + try { + while (state != STATE_RECYCLED) { + final long remaining = deadlineNanos - System.nanoTime(); + if (remaining <= 0) { + return false; + } + LockSupport.parkNanos(this, remaining); + if (Thread.interrupted()) { + Thread.currentThread().interrupt(); + return false; + } + } + return true; + } finally { + if (recycleWaiter == current) { + recycleWaiter = null; + } + } + } + + @Override + public void close() { + if (bufferPtr != 0) { + Unsafe.free(bufferPtr, bufferCapacity, MemoryTag.NATIVE_ILP_RSS); + bufferPtr = 0; + bufferCapacity = 0; + } + } + + /** + * Ensures the buffer has at least the specified capacity. + * Grows the buffer if necessary. + * + * @param requiredCapacity minimum required capacity + */ + public void ensureCapacity(int requiredCapacity) { + if (state != STATE_FILLING) { + throw new IllegalStateException("Cannot resize when state is " + stateName(state)); + } + if (requiredCapacity > bufferCapacity) { + int newCapacity = (int) Math.min(Math.max((long) bufferCapacity * 2, requiredCapacity), Integer.MAX_VALUE); + bufferPtr = Unsafe.realloc(bufferPtr, bufferCapacity, newCapacity, MemoryTag.NATIVE_ILP_RSS); + bufferCapacity = newCapacity; + } + } + + /** + * Returns the age of the first row in nanoseconds, or 0 if no rows. + */ + public long getAgeNanos() { + if (rowCount == 0) { + return 0; + } + return System.nanoTime() - firstRowTimeNanos; + } + + /** + * Returns the batch ID for this buffer. + */ + public long getBatchId() { + return batchId; + } + + /** + * Returns the buffer capacity. + */ + public int getBufferCapacity() { + return bufferCapacity; + } + + /** + * Returns the current write position in the buffer. + */ + public int getBufferPos() { + return bufferPos; + } + + /** + * Returns the buffer pointer for writing data. + * Only valid when state is FILLING. + */ + public long getBufferPtr() { + return bufferPtr; + } + + /** + * Returns the number of rows in this buffer. + */ + public int getRowCount() { + return rowCount; + } + + /** + * Returns the current state. + */ + public int getState() { + return state; + } + + /** + * Returns true if the buffer has any data. + */ + public boolean hasData() { + return bufferPos > 0; + } + + /** + * Increments the row count and records the first row time if this is the first row. + */ + public void incrementRowCount() { + if (state != STATE_FILLING) { + throw new IllegalStateException("Cannot increment row count when state is " + stateName(state)); + } + if (rowCount == 0) { + firstRowTimeNanos = System.nanoTime(); + } + rowCount++; + } + + /** + * Returns true if the buffer is in FILLING state (available for writing). + */ + public boolean isFilling() { + return state == STATE_FILLING; + } + + /** + * Returns true if the buffer is currently in use (not available for the user thread). + */ + public boolean isInUse() { + int s = state; + return s == STATE_SEALED || s == STATE_SENDING; + } + + /** + * Returns true if the buffer is in RECYCLED state (available for reset). + */ + public boolean isRecycled() { + return state == STATE_RECYCLED; + } + + /** + * Returns true if the buffer is in SEALED state (ready to send). + */ + public boolean isSealed() { + return state == STATE_SEALED; + } + + /** + * Returns true if the buffer is in SENDING state (being sent by I/O thread). + */ + public boolean isSending() { + return state == STATE_SENDING; + } + + /** + * Marks the buffer as recycled, transitioning from SENDING to RECYCLED. + * This signals to the user thread that the buffer can be reused. + * Only the I/O thread should call this. + * + * @throws IllegalStateException if not in SENDING state + */ + public void markRecycled() { + if (state != STATE_SENDING) { + throw new IllegalStateException("Cannot mark recycled in state " + stateName(state)); + } + state = STATE_RECYCLED; + Thread w = recycleWaiter; + if (w != null) { + LockSupport.unpark(w); + } + } + + /** + * Marks the buffer as being sent, transitioning from SEALED to SENDING. + * Only the I/O thread should call this. + * + * @throws IllegalStateException if not in SEALED state + */ + public void markSending() { + if (state != STATE_SEALED) { + throw new IllegalStateException("Cannot mark sending in state " + stateName(state)); + } + state = STATE_SENDING; + } + + /** + * Resets the buffer to FILLING state, clearing all data. + * Only valid when in RECYCLED state or when the buffer is fresh. + * + * @throws IllegalStateException if in SEALED or SENDING state + */ + public void reset() { + int s = state; + if (s == STATE_SEALED || s == STATE_SENDING) { + throw new IllegalStateException("Cannot reset buffer in state " + stateName(s)); + } + bufferPos = 0; + rowCount = 0; + firstRowTimeNanos = 0; + batchId = nextBatchId.getAndIncrement(); + recycleWaiter = null; + state = STATE_FILLING; + } + + /** + * Rolls back a seal operation, transitioning from SEALED back to FILLING. + *

    + * Used when enqueue fails after a buffer has been sealed but before ownership + * was transferred to the I/O thread. + * + * @throws IllegalStateException if not in SEALED state + */ + public void rollbackSealForRetry() { + if (state != STATE_SEALED) { + throw new IllegalStateException("Cannot rollback seal in state " + stateName(state)); + } + state = STATE_FILLING; + } + + /** + * Seals the buffer, transitioning from FILLING to SEALED. + * After sealing, no more data can be written. + * Only the user thread should call this. + * + * @throws IllegalStateException if not in FILLING state + */ + public void seal() { + if (state != STATE_FILLING) { + throw new IllegalStateException("Cannot seal buffer in state " + stateName(state)); + } + state = STATE_SEALED; + } + + /** + * Sets the buffer position after external writes. + * Only valid when state is FILLING. + * + * @param pos new position + */ + public void setBufferPos(int pos) { + if (state != STATE_FILLING) { + throw new IllegalStateException("Cannot set position when state is " + stateName(state)); + } + if (pos < 0 || pos > bufferCapacity) { + throw new IllegalArgumentException("Position out of bounds: " + pos); + } + this.bufferPos = pos; + } + + @Override + public String toString() { + return "MicrobatchBuffer{" + + "batchId=" + batchId + + ", state=" + stateName(state) + + ", rows=" + rowCount + + ", bytes=" + bufferPos + + ", capacity=" + bufferCapacity + + '}'; + } + + /** + * Writes bytes to the buffer at the current position. + * Grows the buffer if necessary. + * + * @param src source address + * @param length number of bytes to write + */ + public void write(long src, int length) { + if (state != STATE_FILLING) { + throw new IllegalStateException("Cannot write when state is " + stateName(state)); + } + ensureCapacity((int) Math.min((long) bufferPos + length, Integer.MAX_VALUE)); + Unsafe.getUnsafe().copyMemory(src, bufferPtr + bufferPos, length); + bufferPos += length; + } + + /** + * Writes a single byte to the buffer. + * + * @param b byte to write + */ + public void writeByte(byte b) { + if (state != STATE_FILLING) { + throw new IllegalStateException("Cannot write when state is " + stateName(state)); + } + ensureCapacity((int) Math.min((long) bufferPos + 1, Integer.MAX_VALUE)); + Unsafe.getUnsafe().putByte(bufferPtr + bufferPos, b); + bufferPos++; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java new file mode 100644 index 0000000..18cafdf --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java @@ -0,0 +1,326 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; + +/** + * A simple native memory buffer writer for encoding QWP v1 messages. + *

    + * This class provides write methods similar to HttpClient.Request but writes + * to a native memory buffer that can be sent over WebSocket. + *

    + * All multi-byte values are written in little-endian format unless otherwise specified. + */ +public class NativeBufferWriter implements QwpBufferWriter, QuietCloseable { + + private static final int DEFAULT_CAPACITY = 8192; + + private long bufferPtr; + private int capacity; + private int position; + + public NativeBufferWriter() { + this(DEFAULT_CAPACITY); + } + + public NativeBufferWriter(int initialCapacity) { + this.capacity = initialCapacity; + this.bufferPtr = Unsafe.malloc(capacity, MemoryTag.NATIVE_DEFAULT); + this.position = 0; + } + + /** + * Returns the UTF-8 encoded length of a string. + * + * @param s the string (may be null) + * @return the number of bytes needed to encode the string as UTF-8 + */ + public static int utf8Length(String s) { + if (s == null) return 0; + int len = 0; + for (int i = 0, n = s.length(); i < n; i++) { + char c = s.charAt(i); + if (c < 0x80) { + len++; + } else if (c < 0x800) { + len += 2; + } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n && Character.isLowSurrogate(s.charAt(i + 1))) { + i++; + len += 4; + } else if (Character.isSurrogate(c)) { + len++; + } else { + len += 3; + } + } + return len; + } + + /** + * Returns the number of bytes required to encode {@code value} as an + * unsigned LEB128 varint. + */ + public static int varintSize(long value) { + if (value == 0) { + return 1; + } + return (64 - Long.numberOfLeadingZeros(value) + 6) / 7; + } + + @Override + public void close() { + if (bufferPtr != 0) { + Unsafe.free(bufferPtr, capacity, MemoryTag.NATIVE_DEFAULT); + bufferPtr = 0; + } + } + + /** + * Ensures the buffer has at least the specified additional capacity. + * + * @param needed additional bytes needed beyond current position + */ + @Override + public void ensureCapacity(int needed) { + if (position + needed > capacity) { + int newCapacity = Math.max(capacity * 2, position + needed); + bufferPtr = Unsafe.realloc(bufferPtr, capacity, newCapacity, MemoryTag.NATIVE_DEFAULT); + capacity = newCapacity; + } + } + + /** + * Returns the buffer pointer. + */ + @Override + public long getBufferPtr() { + return bufferPtr; + } + + /** + * Returns the current buffer capacity. + */ + @Override + public int getCapacity() { + return capacity; + } + + /** + * Returns the current write position (number of bytes written). + */ + @Override + public int getPosition() { + return position; + } + + /** + * Patches an int value at the specified offset. + * Used for updating length fields after writing content. + */ + @Override + public void patchInt(int offset, int value) { + Unsafe.getUnsafe().putInt(bufferPtr + offset, value); + } + + /** + * Writes a block of bytes from native memory. + */ + @Override + public void putBlockOfBytes(long from, long len) { + if (len < 0 || len > Integer.MAX_VALUE) { + throw new IllegalArgumentException("len exceeds int range: " + len); + } + int intLen = (int) len; + ensureCapacity(intLen); + Unsafe.getUnsafe().copyMemory(from, bufferPtr + position, intLen); + position += intLen; + } + + /** + * Writes a single byte. + */ + @Override + public void putByte(byte value) { + ensureCapacity(1); + Unsafe.getUnsafe().putByte(bufferPtr + position, value); + position++; + } + + /** + * Writes a double (8 bytes, little-endian). + */ + @Override + public void putDouble(double value) { + ensureCapacity(8); + Unsafe.getUnsafe().putDouble(bufferPtr + position, value); + position += 8; + } + + /** + * Writes a float (4 bytes, little-endian). + */ + @Override + public void putFloat(float value) { + ensureCapacity(4); + Unsafe.getUnsafe().putFloat(bufferPtr + position, value); + position += 4; + } + + /** + * Writes an int (4 bytes, little-endian). + */ + @Override + public void putInt(int value) { + ensureCapacity(4); + Unsafe.getUnsafe().putInt(bufferPtr + position, value); + position += 4; + } + + /** + * Writes a long (8 bytes, little-endian). + */ + @Override + public void putLong(long value) { + ensureCapacity(8); + Unsafe.getUnsafe().putLong(bufferPtr + position, value); + position += 8; + } + + /** + * Writes a long in big-endian order. + */ + @Override + public void putLongBE(long value) { + ensureCapacity(8); + Unsafe.getUnsafe().putLong(bufferPtr + position, Long.reverseBytes(value)); + position += 8; + } + + /** + * Writes a short (2 bytes, little-endian). + */ + @Override + public void putShort(short value) { + ensureCapacity(2); + Unsafe.getUnsafe().putShort(bufferPtr + position, value); + position += 2; + } + + /** + * Writes a length-prefixed UTF-8 string. + */ + @Override + public void putString(String value) { + if (value == null || value.isEmpty()) { + putVarint(0); + return; + } + + int utf8Len = utf8Length(value); + putVarint(utf8Len); + ensureCapacity(utf8Len); + encodeUtf8(value); + } + + /** + * Writes UTF-8 bytes directly without length prefix. + */ + @Override + public void putUtf8(String value) { + if (value == null || value.isEmpty()) { + return; + } + int utf8Len = utf8Length(value); + ensureCapacity(utf8Len); + encodeUtf8(value); + } + + /** + * Writes a varint (unsigned LEB128). + */ + @Override + public void putVarint(long value) { + while (value > 0x7F) { + putByte((byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + putByte((byte) value); + } + + /** + * Resets the buffer for reuse. + */ + @Override + public void reset() { + position = 0; + } + + /** + * Skips the specified number of bytes, advancing the position. + * Used when data has been written directly to the buffer via getBufferPtr(). + * + * @param bytes number of bytes to skip + */ + @Override + public void skip(int bytes) { + ensureCapacity(bytes); + position += bytes; + } + + private void encodeUtf8(String value) { + long addr = bufferPtr + position; + for (int i = 0, n = value.length(); i < n; i++) { + char c = value.charAt(i); + if (c < 0x80) { + Unsafe.getUnsafe().putByte(addr++, (byte) c); + } else if (c < 0x800) { + Unsafe.getUnsafe().putByte(addr++, (byte) (0xC0 | (c >> 6))); + Unsafe.getUnsafe().putByte(addr++, (byte) (0x80 | (c & 0x3F))); + } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n) { + char c2 = value.charAt(++i); + if (Character.isLowSurrogate(c2)) { + int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); + Unsafe.getUnsafe().putByte(addr++, (byte) (0xF0 | (codePoint >> 18))); + Unsafe.getUnsafe().putByte(addr++, (byte) (0x80 | ((codePoint >> 12) & 0x3F))); + Unsafe.getUnsafe().putByte(addr++, (byte) (0x80 | ((codePoint >> 6) & 0x3F))); + Unsafe.getUnsafe().putByte(addr++, (byte) (0x80 | (codePoint & 0x3F))); + } else { + Unsafe.getUnsafe().putByte(addr++, (byte) '?'); + i--; + } + } else if (Character.isSurrogate(c)) { + Unsafe.getUnsafe().putByte(addr++, (byte) '?'); + } else { + Unsafe.getUnsafe().putByte(addr++, (byte) (0xE0 | (c >> 12))); + Unsafe.getUnsafe().putByte(addr++, (byte) (0x80 | ((c >> 6) & 0x3F))); + Unsafe.getUnsafe().putByte(addr++, (byte) (0x80 | (c & 0x3F))); + } + } + position = (int) (addr - bufferPtr); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeSegmentList.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeSegmentList.java new file mode 100644 index 0000000..91d4eab --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeSegmentList.java @@ -0,0 +1,123 @@ +/* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; + +final class NativeSegmentList implements QuietCloseable { + static final int ENTRY_SIZE = 16; + + private int capacity; + private long ptr; + private int size; + private long totalLength; + + NativeSegmentList() { + this(16); + } + + NativeSegmentList(int initialCapacity) { + this.capacity = Math.max(initialCapacity, 4); + this.ptr = Unsafe.malloc((long) capacity * ENTRY_SIZE, MemoryTag.NATIVE_DEFAULT); + } + + @Override + public void close() { + if (ptr != 0) { + Unsafe.free(ptr, (long) capacity * ENTRY_SIZE, MemoryTag.NATIVE_DEFAULT); + ptr = 0; + capacity = 0; + size = 0; + totalLength = 0; + } + } + + private void ensureCapacity(int required) { + if (required <= capacity) { + return; + } + + int newCapacity = capacity; + while (newCapacity < required) { + if (newCapacity > Integer.MAX_VALUE / 2) { + newCapacity = required; + break; + } + newCapacity *= 2; + } + + ptr = Unsafe.realloc( + ptr, + (long) capacity * ENTRY_SIZE, + (long) newCapacity * ENTRY_SIZE, + MemoryTag.NATIVE_DEFAULT + ); + capacity = newCapacity; + } + + void add(long address, long length) { + if (length <= 0) { + return; + } + ensureCapacity(size + 1); + long segmentPtr = ptr + (long) size * ENTRY_SIZE; + Unsafe.getUnsafe().putLong(segmentPtr, address); + Unsafe.getUnsafe().putLong(segmentPtr + 8, length); + size++; + totalLength += length; + } + + void appendFrom(NativeSegmentList other) { + if (other.size == 0) { + return; + } + ensureCapacity(size + other.size); + Unsafe.getUnsafe().copyMemory( + other.ptr, + ptr + (long) size * ENTRY_SIZE, + (long) other.size * ENTRY_SIZE + ); + size += other.size; + totalLength += other.totalLength; + } + + long getAddress() { + return ptr; + } + + int getSegmentCount() { + return size; + } + + long getTotalLength() { + return totalLength; + } + + void reset() { + size = 0; + totalLength = 0; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java new file mode 100644 index 0000000..0b8101c --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java @@ -0,0 +1,138 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.cutlass.line.array.ArrayBufferAppender; + +/** + * Buffer writer interface for QWP v1 message encoding. + *

    + * This interface extends {@link ArrayBufferAppender} with additional methods + * required for encoding QWP v1 messages, including varint encoding, string + * handling, and buffer manipulation. + *

    + * Implementations include: + *

      + *
    • {@link NativeBufferWriter} - standalone native memory buffer
      • + *
    • {@link io.questdb.client.cutlass.http.client.WebSocketSendBuffer} - WebSocket frame buffer
      • + *
    + *

    + * All multi-byte values are written in little-endian format unless the method + * name explicitly indicates big-endian (e.g., {@link #putLongBE}). + */ +public interface QwpBufferWriter extends ArrayBufferAppender { + + /** + * Ensures the buffer has capacity for at least the specified + * additional bytes beyond the current position. + * + * @param additionalBytes number of additional bytes needed + */ + void ensureCapacity(int additionalBytes); + + /** + * Returns the native memory pointer to the buffer start. + *

    + * The returned pointer is valid until the next buffer growth operation. + * Use with care and only for reading completed data. + */ + long getBufferPtr(); + + /** + * Returns the current buffer capacity in bytes. + */ + int getCapacity(); + + /** + * Returns the current write position (number of bytes written). + */ + int getPosition(); + + /** + * Patches an int value at the specified offset in the buffer. + *

    + * Used for updating length fields after writing content. + * + * @param offset the byte offset from buffer start + * @param value the int value to write + */ + void patchInt(int offset, int value); + + /** + * Writes a float (4 bytes, little-endian). + */ + void putFloat(float value); + + /** + * Writes a long in big-endian byte order. + */ + void putLongBE(long value); + + /** + * Writes a short (2 bytes, little-endian). + */ + void putShort(short value); + + /** + * Writes a length-prefixed UTF-8 string. + *

    + * Format: varint length + UTF-8 bytes + * + * @param value the string to write (may be null or empty) + */ + void putString(String value); + + /** + * Writes UTF-8 encoded bytes directly without length prefix. + * + * @param value the string to encode (may be null or empty) + */ + void putUtf8(String value); + + /** + * Writes an unsigned variable-length integer (LEB128 encoding). + *

    + * Each byte contains 7 bits of data with the high bit indicating + * whether more bytes follow. + */ + void putVarint(long value); + + /** + * Resets the buffer for reuse, setting the position to 0. + *

    + * Does not deallocate memory. + */ + void reset(); + + /** + * Skips the specified number of bytes, advancing the position. + *

    + * Used when data has been written directly to the buffer via + * {@link #getBufferPtr()}. + * + * @param bytes number of bytes to skip + */ + void skip(int bytes); +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java new file mode 100644 index 0000000..e07bd7f --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java @@ -0,0 +1,371 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.protocol.QwpColumnDef; +import io.questdb.client.cutlass.qwp.protocol.QwpGorillaEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.Unsafe; + +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; + +/** + * Transport-agnostic column encoder for ILP v4 table data. + *

    + * Reads column data from {@link QwpTableBuffer.ColumnBuffer} and writes encoded + * bytes to a {@link QwpBufferWriter}. Both {@link QwpWebSocketEncoder} and + * {@link QwpUdpSender} delegate to this class for column encoding. + */ +class QwpColumnWriter { + + private static final byte ENCODING_GORILLA = 0x01; + private static final byte ENCODING_UNCOMPRESSED = 0x00; + private final QwpGorillaEncoder gorillaEncoder = new QwpGorillaEncoder(); + private QwpBufferWriter buffer; + + private void encodeColumn( + QwpTableBuffer.ColumnBuffer col, + QwpColumnDef colDef, + int rowCount, + int valueCount, + long stringDataSize, + int symbolDictionarySize, + boolean useGorilla, + boolean useGlobalSymbols + ) { + long dataAddr = col.getDataAddress(); + + writeNullHeader(col, rowCount, rowCount - valueCount); + + switch (col.getType()) { + case TYPE_BOOLEAN: + writeBooleanColumn(dataAddr, valueCount); + break; + case TYPE_BYTE: + buffer.putBlockOfBytes(dataAddr, valueCount); + break; + case TYPE_SHORT: + case TYPE_CHAR: + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 2); + break; + case TYPE_INT: + case TYPE_FLOAT: + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 4); + break; + case TYPE_LONG: + case TYPE_DATE: + case TYPE_DOUBLE: + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 8); + break; + case TYPE_TIMESTAMP: + case TYPE_TIMESTAMP_NANOS: + writeTimestampColumn(dataAddr, valueCount, useGorilla); + break; + case TYPE_GEOHASH: + writeGeoHashColumn(dataAddr, valueCount, col.getGeoHashPrecision()); + break; + case TYPE_STRING: + case TYPE_VARCHAR: + writeStringColumn(col, valueCount, stringDataSize); + break; + case TYPE_SYMBOL: + if (useGlobalSymbols) { + writeSymbolColumnWithGlobalIds(col, valueCount); + } else { + writeSymbolColumn(col, valueCount, symbolDictionarySize); + } + break; + case TYPE_UUID: + // Stored as lo+hi contiguously, matching wire order + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 16); + break; + case TYPE_LONG256: + // Stored as 4 contiguous longs per value + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 32); + break; + case TYPE_DOUBLE_ARRAY: + writeDoubleArrayColumn(col, valueCount); + break; + case TYPE_LONG_ARRAY: + writeLongArrayColumn(col, valueCount); + break; + case TYPE_DECIMAL64: + writeDecimal64Column(col.getDecimalScale(), dataAddr, valueCount); + break; + case TYPE_DECIMAL128: + writeDecimal128Column(col.getDecimalScale(), dataAddr, valueCount); + break; + case TYPE_DECIMAL256: + writeDecimal256Column(col.getDecimalScale(), dataAddr, valueCount); + break; + default: + throw new LineSenderException("Unknown column type: " + col.getType()); + } + } + + void encodeTable(QwpTableBuffer tableBuffer, boolean useSchemaRef, boolean useGlobalSymbols, boolean useGorilla) { + encodeTable(tableBuffer, tableBuffer.getRowCount(), null, null, null, useSchemaRef, useGlobalSymbols, useGorilla); + } + + void encodeTable( + QwpTableBuffer tableBuffer, + int rowCount, + int[] limitedValueCounts, + long[] limitedStringDataSizes, + int[] limitedSymbolDictionarySizes, + boolean useSchemaRef, + boolean useGlobalSymbols, + boolean useGorilla + ) { + QwpColumnDef[] columnDefs = tableBuffer.getColumnDefs(); + + if (useSchemaRef) { + writeTableHeaderWithSchemaRef( + tableBuffer.getTableName(), + rowCount, + tableBuffer.getSchemaHash(), + columnDefs.length + ); + } else { + writeTableHeaderWithSchema(tableBuffer.getTableName(), rowCount, columnDefs); + } + + for (int i = 0; i < tableBuffer.getColumnCount(); i++) { + QwpTableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); + QwpColumnDef colDef = columnDefs[i]; + int valueCount = col.getValueCount(); + long stringDataSize = col.getStringDataSize(); + int symbolDictionarySize = col.getSymbolDictionarySize(); + + if (limitedValueCounts != null && limitedValueCounts[i] > -1) { + valueCount = limitedValueCounts[i]; + stringDataSize = limitedStringDataSizes[i]; + symbolDictionarySize = limitedSymbolDictionarySizes[i]; + } + + encodeColumn(col, colDef, rowCount, valueCount, stringDataSize, symbolDictionarySize, useGorilla, useGlobalSymbols); + } + } + + void setBuffer(QwpBufferWriter buffer) { + this.buffer = buffer; + } + + private void writeBooleanColumn(long addr, int count) { + int packedSize = (count + 7) / 8; + for (int i = 0; i < packedSize; i++) { + byte b = 0; + for (int bit = 0; bit < 8; bit++) { + int idx = i * 8 + bit; + if (idx < count && Unsafe.getUnsafe().getByte(addr + idx) != 0) { + b |= (1 << bit); + } + } + buffer.putByte(b); + } + } + + private void writeDecimal128Column(byte scale, long addr, int count) { + buffer.putByte(scale); + for (int i = 0; i < count; i++) { + long offset = (long) i * 16; + long hi = Unsafe.getUnsafe().getLong(addr + offset); + long lo = Unsafe.getUnsafe().getLong(addr + offset + 8); + buffer.putLongBE(hi); + buffer.putLongBE(lo); + } + } + + private void writeDecimal256Column(byte scale, long addr, int count) { + buffer.putByte(scale); + for (int i = 0; i < count; i++) { + long offset = (long) i * 32; + buffer.putLongBE(Unsafe.getUnsafe().getLong(addr + offset)); + buffer.putLongBE(Unsafe.getUnsafe().getLong(addr + offset + 8)); + buffer.putLongBE(Unsafe.getUnsafe().getLong(addr + offset + 16)); + buffer.putLongBE(Unsafe.getUnsafe().getLong(addr + offset + 24)); + } + } + + private void writeDecimal64Column(byte scale, long addr, int count) { + buffer.putByte(scale); + for (int i = 0; i < count; i++) { + buffer.putLongBE(Unsafe.getUnsafe().getLong(addr + (long) i * 8)); + } + } + + private void writeDoubleArrayColumn(QwpTableBuffer.ColumnBuffer col, int count) { + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + double[] data = col.getDoubleArrayData(); + + int shapeIdx = 0; + int dataIdx = 0; + for (int row = 0; row < count; row++) { + int nDims = dims[row]; + buffer.putByte((byte) nDims); + + int elemCount = 1; + for (int d = 0; d < nDims; d++) { + int dimLen = shapes[shapeIdx++]; + buffer.putInt(dimLen); + elemCount = Math.multiplyExact(elemCount, dimLen); + } + + for (int e = 0; e < elemCount; e++) { + buffer.putDouble(data[dataIdx++]); + } + } + } + + private void writeGeoHashColumn(long addr, int count, int precision) { + if (precision < 1) { + precision = 1; + } + buffer.putVarint(precision); + int valueSize = (precision + 7) / 8; + for (int i = 0; i < count; i++) { + long value = Unsafe.getUnsafe().getLong(addr + (long) i * 8); + for (int b = 0; b < valueSize; b++) { + buffer.putByte((byte) (value >>> (b * 8))); + } + } + } + + private void writeLongArrayColumn(QwpTableBuffer.ColumnBuffer col, int count) { + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + long[] data = col.getLongArrayData(); + + int shapeIdx = 0; + int dataIdx = 0; + for (int row = 0; row < count; row++) { + int nDims = dims[row]; + buffer.putByte((byte) nDims); + + int elemCount = 1; + for (int d = 0; d < nDims; d++) { + int dimLen = shapes[shapeIdx++]; + buffer.putInt(dimLen); + elemCount = Math.multiplyExact(elemCount, dimLen); + } + + for (int e = 0; e < elemCount; e++) { + buffer.putLong(data[dataIdx++]); + } + } + } + + private void writeNullHeader(QwpTableBuffer.ColumnBuffer col, int rowCount, int nullCount) { + if (nullCount > 0) { + buffer.putByte((byte) 1); + col.ensureNullBitmapCapacity(rowCount); + long nullAddr = col.getNullBitmapAddress(); + int bitmapSize = (rowCount + 7) / 8; + buffer.putBlockOfBytes(nullAddr, bitmapSize); + } else { + buffer.putByte((byte) 0); + } + } + + private void writeStringColumn(QwpTableBuffer.ColumnBuffer col, int valueCount, long stringDataSize) { + buffer.putBlockOfBytes(col.getStringOffsetsAddress(), (long) (valueCount + 1) * 4); + buffer.putBlockOfBytes(col.getStringDataAddress(), stringDataSize); + } + + private void writeSymbolColumn(QwpTableBuffer.ColumnBuffer col, int count, int dictionarySize) { + long dataAddr = col.getDataAddress(); + buffer.putVarint(dictionarySize); + for (int i = 0; i < dictionarySize; i++) { + buffer.putString((String) col.getSymbolValue(i)); + } + + for (int i = 0; i < count; i++) { + int idx = Unsafe.getUnsafe().getInt(dataAddr + (long) i * 4); + buffer.putVarint(idx); + } + } + + private void writeSymbolColumnWithGlobalIds(QwpTableBuffer.ColumnBuffer col, int count) { + long auxAddr = col.getAuxDataAddress(); + if (auxAddr == 0) { + long dataAddr = col.getDataAddress(); + for (int i = 0; i < count; i++) { + int idx = Unsafe.getUnsafe().getInt(dataAddr + (long) i * 4); + buffer.putVarint(idx); + } + } else { + for (int i = 0; i < count; i++) { + int globalId = Unsafe.getUnsafe().getInt(auxAddr + (long) i * 4); + buffer.putVarint(globalId); + } + } + } + + private void writeTableHeaderWithSchema(String tableName, int rowCount, QwpColumnDef[] columns) { + buffer.putString(tableName); + buffer.putVarint(rowCount); + buffer.putVarint(columns.length); + buffer.putByte(SCHEMA_MODE_FULL); + for (QwpColumnDef col : columns) { + buffer.putString(col.getName()); + buffer.putByte(col.getWireTypeCode()); + } + } + + private void writeTableHeaderWithSchemaRef(String tableName, int rowCount, long schemaHash, int columnCount) { + buffer.putString(tableName); + buffer.putVarint(rowCount); + buffer.putVarint(columnCount); + buffer.putByte(SCHEMA_MODE_REFERENCE); + buffer.putLong(schemaHash); + } + + private void writeTimestampColumn(long addr, int count, boolean useGorilla) { + if (useGorilla && count > 2) { + if (QwpGorillaEncoder.canUseGorilla(addr, count)) { + buffer.putByte(ENCODING_GORILLA); + int encodedSize = QwpGorillaEncoder.calculateEncodedSize(addr, count); + buffer.ensureCapacity(encodedSize); + int bytesWritten = gorillaEncoder.encodeTimestamps( + buffer.getBufferPtr() + buffer.getPosition(), + buffer.getCapacity() - buffer.getPosition(), + addr, + count + ); + buffer.skip(bytesWritten); + } else { + buffer.putByte(ENCODING_UNCOMPRESSED); + buffer.putBlockOfBytes(addr, (long) count * 8); + } + } else { + if (useGorilla) { + buffer.putByte(ENCODING_UNCOMPRESSED); + } + buffer.putBlockOfBytes(addr, (long) count * 8); + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java new file mode 100644 index 0000000..181d4b2 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java @@ -0,0 +1,1429 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.line.array.DoubleArray; +import io.questdb.client.cutlass.line.array.LongArray; +import io.questdb.client.cutlass.line.udp.UdpLineChannel; +import io.questdb.client.cutlass.qwp.protocol.QwpColumnDef; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.network.NetworkFacade; +import io.questdb.client.std.CharSequenceObjHashMap; +import io.questdb.client.std.Chars; +import io.questdb.client.std.Decimal128; +import io.questdb.client.std.Decimal256; +import io.questdb.client.std.Decimal64; +import io.questdb.client.std.Misc; +import io.questdb.client.std.ObjList; +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.bytes.DirectByteSlice; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.TestOnly; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Arrays; + +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; + +/** + * Fire-and-forget ILP v4 sender over UDP. + *

    + * Each {@link #flush()} encodes all buffered table data into self-contained + * datagrams (one per table) and sends them via UDP. Datagrams use local + * symbol dictionaries (no global/delta dict) and full schema (no schema refs). + *

    + * When {@code maxDatagramSize > 0}, the sender automatically flushes before + * a datagram exceeds the size limit. The in-progress row stays staged in sender + * state until commit, so committed table data can be flushed without replaying + * the row back into column storage. + */ +public class QwpUdpSender implements Sender { + private static final int ADAPTIVE_HEADROOM_EWMA_SHIFT = 2; + private static final Logger LOG = LoggerFactory.getLogger(QwpUdpSender.class); + private static final int VARINT_INT_UPPER_BOUND = 5; + private final UdpLineChannel channel; + private final QwpColumnWriter columnWriter = new QwpColumnWriter(); + private final NativeSegmentList datagramSegments; + private final NativeBufferWriter headerBuffer; + private final int maxDatagramSize; + private final SegmentedNativeBufferWriter payloadWriter; + private final CharSequenceObjHashMap tableBuffers; + private final CharSequenceObjHashMap tableHeadroomStates; + private final boolean trackDatagramEstimate; + private QwpTableBuffer.ColumnBuffer cachedTimestampColumn; + private QwpTableBuffer.ColumnBuffer cachedTimestampNanosColumn; + private boolean closed; + private long committedDatagramEstimate; + // monotonically increasing mark; incremented per row to invalidate stagedColumnMarks without clearing + private int currentRowMark = 1; + private QwpTableBuffer currentTableBuffer; + private TableHeadroomState currentTableHeadroomState; + private String currentTableName; + private int inProgressColumnCount; + private InProgressColumnState[] inProgressColumns = new InProgressColumnState[8]; + private int inProgressRowValueCount; + // prefix* arrays: per-column snapshots captured before the in-progress row, + // used to encode and flush only the committed prefix when a row is still being built. + // Indexed by column index. -1 means the column has no in-progress data. + private int[] prefixArrayDataOffsetBefore = new int[8]; + private int[] prefixArrayShapeOffsetBefore = new int[8]; + private int[] prefixSizeBefore = new int[8]; + private long[] prefixStringDataSizeBefore = new long[8]; + private int[] prefixSymbolDictionarySizeBefore = new int[8]; + private int[] prefixValueCountBefore = new int[8]; + // per-column marks to detect duplicate writes within a single row; compared against currentRowMark + private int[] stagedColumnMarks = new int[8]; + + public QwpUdpSender(NetworkFacade nf, int interfaceIPv4, int sendToAddress, int port, int ttl) { + this(nf, interfaceIPv4, sendToAddress, port, ttl, 0); + } + + public QwpUdpSender(NetworkFacade nf, int interfaceIPv4, int sendToAddress, int port, int ttl, int maxDatagramSize) { + NativeSegmentList segments = null; + NativeBufferWriter header = null; + SegmentedNativeBufferWriter payload = null; + UdpLineChannel ch = null; + try { + segments = new NativeSegmentList(); + header = new NativeBufferWriter(); + payload = new SegmentedNativeBufferWriter(); + ch = new UdpLineChannel(nf, interfaceIPv4, sendToAddress, port, ttl); + this.tableHeadroomStates = new CharSequenceObjHashMap<>(); + this.tableBuffers = new CharSequenceObjHashMap<>(); + } catch (Throwable t) { + Misc.free(ch); + Misc.free(payload); + Misc.free(header); + Misc.free(segments); + throw t; + } + this.channel = ch; + this.datagramSegments = segments; + this.headerBuffer = header; + this.payloadWriter = payload; + this.maxDatagramSize = maxDatagramSize; + this.trackDatagramEstimate = maxDatagramSize > 0; + } + + @Override + public void at(long timestamp, ChronoUnit unit) { + checkNotClosed(); + checkTableSelected(); + if (unit == ChronoUnit.NANOS) { + atNanos(timestamp); + } else { + long micros = toMicros(timestamp, unit); + atMicros(micros); + } + } + + @Override + public void at(Instant timestamp) { + checkNotClosed(); + checkTableSelected(); + long micros = timestamp.getEpochSecond() * 1_000_000L + timestamp.getNano() / 1000L; + atMicros(micros); + } + + @Override + public void atNow() { + checkNotClosed(); + checkTableSelected(); + try { + commitCurrentRow(); + } catch (RuntimeException | Error e) { + rollbackCurrentRowToCommittedState(); + throw e; + } + } + + @Override + public Sender boolColumn(CharSequence columnName, boolean value) { + checkNotClosed(); + checkTableSelected(); + try { + stageBooleanColumnValue(columnName, value); + } catch (RuntimeException | Error e) { + rollbackCurrentRowToCommittedState(); + throw e; + } + return this; + } + + @Override + public DirectByteSlice bufferView() { + throw new LineSenderException("bufferView() is not supported for UDP sender"); + } + + @Override + public void cancelRow() { + checkNotClosed(); + rollbackCurrentRowToCommittedState(); + } + + @Override + public void close() { + if (!closed) { + try { + if (hasInProgressRow()) { + rollbackCurrentRowToCommittedState(); + } + flushInternal(); + } catch (Exception e) { + LOG.error("Error during close flush: {}", String.valueOf(e)); + } + closed = true; + ObjList keys = tableBuffers.keys(); + for (int i = 0, n = keys.size(); i < n; i++) { + CharSequence key = keys.getQuick(i); + if (key != null) { + QwpTableBuffer tb = tableBuffers.get(key); + if (tb != null) { + tb.close(); + } + } + } + tableBuffers.clear(); + tableHeadroomStates.clear(); + currentTableHeadroomState = null; + channel.close(); + payloadWriter.close(); + datagramSegments.close(); + headerBuffer.close(); + } + } + + @TestOnly + public long committedDatagramEstimateForTest() { + return committedDatagramEstimate; + } + + @TestOnly + public QwpTableBuffer currentTableBufferForTest() { + return currentTableBuffer; + } + + @Override + public Sender decimalColumn(CharSequence name, Decimal64 value) { + if (value == null || value.isNull()) { + return this; + } + checkNotClosed(); + checkTableSelected(); + try { + stageDecimal64ColumnValue(name, value); + } catch (RuntimeException | Error e) { + rollbackCurrentRowToCommittedState(); + throw e; + } + return this; + } + + @Override + public Sender decimalColumn(CharSequence name, Decimal128 value) { + if (value == null || value.isNull()) { + return this; + } + checkNotClosed(); + checkTableSelected(); + try { + stageDecimal128ColumnValue(name, value); + } catch (RuntimeException | Error e) { + rollbackCurrentRowToCommittedState(); + throw e; + } + return this; + } + + @Override + public Sender decimalColumn(CharSequence name, Decimal256 value) { + if (value == null || value.isNull()) { + return this; + } + checkNotClosed(); + checkTableSelected(); + try { + stageDecimal256ColumnValue(name, value); + } catch (RuntimeException | Error e) { + rollbackCurrentRowToCommittedState(); + throw e; + } + return this; + } + + @Override + public Sender doubleArray(@NotNull CharSequence name, double[] values) { + if (values == null) { + return this; + } + checkNotClosed(); + checkTableSelected(); + try { + stageDoubleArrayColumnValue(name, values); + } catch (RuntimeException | Error e) { + rollbackCurrentRowToCommittedState(); + throw e; + } + return this; + } + + @Override + public Sender doubleArray(@NotNull CharSequence name, double[][] values) { + if (values == null) { + return this; + } + checkNotClosed(); + checkTableSelected(); + try { + stageDoubleArrayColumnValue(name, values); + } catch (RuntimeException | Error e) { + rollbackCurrentRowToCommittedState(); + throw e; + } + return this; + } + + @Override + public Sender doubleArray(@NotNull CharSequence name, double[][][] values) { + if (values == null) { + return this; + } + checkNotClosed(); + checkTableSelected(); + try { + stageDoubleArrayColumnValue(name, values); + } catch (RuntimeException | Error e) { + rollbackCurrentRowToCommittedState(); + throw e; + } + return this; + } + + @Override + public Sender doubleArray(CharSequence name, DoubleArray array) { + if (array == null) { + return this; + } + checkNotClosed(); + checkTableSelected(); + try { + stageDoubleArrayColumnValue(name, array); + } catch (RuntimeException | Error e) { + rollbackCurrentRowToCommittedState(); + throw e; + } + return this; + } + + @Override + public Sender doubleColumn(CharSequence columnName, double value) { + checkNotClosed(); + checkTableSelected(); + try { + stageDoubleColumnValue(columnName, value); + } catch (RuntimeException | Error e) { + rollbackCurrentRowToCommittedState(); + throw e; + } + return this; + } + + @Override + public void flush() { + checkNotClosed(); + ensureNoInProgressRow("flush buffer"); + flushInternal(); + } + + @Override + public Sender longArray(@NotNull CharSequence name, long[] values) { + if (values == null) { + return this; + } + checkNotClosed(); + checkTableSelected(); + try { + stageLongArrayColumnValue(name, values); + } catch (RuntimeException | Error e) { + rollbackCurrentRowToCommittedState(); + throw e; + } + return this; + } + + @Override + public Sender longArray(@NotNull CharSequence name, long[][] values) { + if (values == null) { + return this; + } + checkNotClosed(); + checkTableSelected(); + try { + stageLongArrayColumnValue(name, values); + } catch (RuntimeException | Error e) { + rollbackCurrentRowToCommittedState(); + throw e; + } + return this; + } + + @Override + public Sender longArray(@NotNull CharSequence name, long[][][] values) { + if (values == null) { + return this; + } + checkNotClosed(); + checkTableSelected(); + try { + stageLongArrayColumnValue(name, values); + } catch (RuntimeException | Error e) { + rollbackCurrentRowToCommittedState(); + throw e; + } + return this; + } + + @Override + public Sender longArray(@NotNull CharSequence name, LongArray array) { + if (array == null) { + return this; + } + checkNotClosed(); + checkTableSelected(); + try { + stageLongArrayColumnValue(name, array); + } catch (RuntimeException | Error e) { + rollbackCurrentRowToCommittedState(); + throw e; + } + return this; + } + + @Override + public Sender longColumn(CharSequence columnName, long value) { + checkNotClosed(); + checkTableSelected(); + try { + stageLongColumnValue(columnName, value); + } catch (RuntimeException | Error e) { + rollbackCurrentRowToCommittedState(); + throw e; + } + return this; + } + + @Override + public void reset() { + checkNotClosed(); + ObjList keys = tableBuffers.keys(); + for (int i = 0, n = keys.size(); i < n; i++) { + QwpTableBuffer buf = tableBuffers.get(keys.getQuick(i)); + if (buf != null) { + buf.rollbackUncommittedColumns(); + buf.reset(); + } + } + currentTableBuffer = null; + currentTableHeadroomState = null; + currentTableName = null; + clearTransientRowState(); + resetCommittedDatagramEstimate(); + tableHeadroomStates.clear(); + } + + // Public test hooks because module boundaries prevent tests from sharing this package. + @TestOnly + public void stageNullDoubleArrayForTest(CharSequence name) { + stageNullArrayColumnValue(name, TYPE_DOUBLE_ARRAY); + } + + @TestOnly + public void stageNullLongArrayForTest(CharSequence name) { + stageNullArrayColumnValue(name, TYPE_LONG_ARRAY); + } + + @Override + public Sender stringColumn(CharSequence columnName, CharSequence value) { + checkNotClosed(); + checkTableSelected(); + try { + stageStringColumnValue(columnName, value); + } catch (RuntimeException | Error e) { + rollbackCurrentRowToCommittedState(); + throw e; + } + return this; + } + + @Override + public Sender symbol(CharSequence columnName, CharSequence value) { + checkNotClosed(); + checkTableSelected(); + try { + stageSymbolColumnValue(columnName, value); + } catch (RuntimeException | Error e) { + rollbackCurrentRowToCommittedState(); + throw e; + } + return this; + } + + @Override + public Sender table(CharSequence tableName) { + checkNotClosed(); + if (currentTableName != null && currentTableBuffer != null && Chars.equals(tableName, currentTableName)) { + return this; + } + ensureNoInProgressRow("switch tables"); + if (trackDatagramEstimate && currentTableBuffer != null && currentTableBuffer.getRowCount() > 0) { + flushSingleTable(currentTableName, currentTableBuffer); + } else { + clearTransientRowState(); + resetCommittedDatagramEstimate(); + } + + currentTableName = tableName.toString(); + currentTableBuffer = tableBuffers.get(currentTableName); + if (currentTableBuffer == null) { + currentTableBuffer = new QwpTableBuffer(currentTableName); + tableBuffers.put(currentTableName, currentTableBuffer); + } + currentTableHeadroomState = tableHeadroomStates.get(currentTableName); + if (currentTableHeadroomState == null) { + currentTableHeadroomState = new TableHeadroomState(); + tableHeadroomStates.put(currentTableName, currentTableHeadroomState); + } + return this; + } + + @Override + public Sender timestampColumn(CharSequence columnName, long value, ChronoUnit unit) { + checkNotClosed(); + checkTableSelected(); + try { + if (unit == ChronoUnit.NANOS) { + stageTimestampColumnValue(columnName, TYPE_TIMESTAMP_NANOS, value); + } else { + long micros = toMicros(value, unit); + stageTimestampColumnValue(columnName, TYPE_TIMESTAMP, micros); + } + } catch (RuntimeException | Error e) { + rollbackCurrentRowToCommittedState(); + throw e; + } + return this; + } + + @Override + public Sender timestampColumn(CharSequence columnName, Instant value) { + checkNotClosed(); + checkTableSelected(); + try { + long micros = value.getEpochSecond() * 1_000_000L + value.getNano() / 1000L; + stageTimestampColumnValue(columnName, TYPE_TIMESTAMP, micros); + } catch (RuntimeException | Error e) { + rollbackCurrentRowToCommittedState(); + throw e; + } + return this; + } + + private static int bitmapBytes(int size) { + return (size + 7) / 8; + } + + private static long estimateArrayPayloadBytes(QwpTableBuffer.ColumnBuffer col, InProgressColumnState state) { + int shapeCount = col.getArrayShapeOffset() - state.arrayShapeOffsetBefore; + int dataCount = col.getArrayDataOffset() - state.arrayDataOffsetBefore; + int elementSize = col.getType() == TYPE_LONG_ARRAY ? Long.BYTES : Double.BYTES; + return 1L + (long) shapeCount * Integer.BYTES + (long) dataCount * elementSize; + } + + private static long estimateInProgressColumnPayload(InProgressColumnState state) { + QwpTableBuffer.ColumnBuffer col = state.column; + int valueCountBefore = state.valueCountBefore; + int valueCountAfter = col.getValueCount(); + if (valueCountAfter == valueCountBefore) { + return 0; + } + + switch (col.getType()) { + case TYPE_BOOLEAN: + return packedBytes(valueCountAfter) - packedBytes(valueCountBefore); + case TYPE_DECIMAL64: + return 8; + case TYPE_DECIMAL128: + return 16; + case TYPE_DECIMAL256: + return 32; + case TYPE_DOUBLE: + case TYPE_LONG: + case TYPE_TIMESTAMP: + case TYPE_TIMESTAMP_NANOS: + return 8; + case TYPE_DOUBLE_ARRAY: + case TYPE_LONG_ARRAY: + return estimateArrayPayloadBytes(col, state); + case TYPE_STRING: + case TYPE_VARCHAR: + return 4L + (col.getStringDataSize() - state.stringDataSizeBefore); + case TYPE_SYMBOL: + return estimateSymbolPayloadDelta(col, state); + default: + throw new LineSenderException("unsupported in-progress column type: " + col.getType()); + } + } + + private static long estimateSymbolPayloadDelta(QwpTableBuffer.ColumnBuffer col, InProgressColumnState state) { + int valueCountBefore = state.valueCountBefore; + int valueCountAfter = col.getValueCount(); + if (valueCountAfter == valueCountBefore) { + return 0; + } + + int dictSizeBefore = state.symbolDictionarySizeBefore; + long dataAddress = col.getDataAddress(); + int idx = Unsafe.getUnsafe().getInt(dataAddress + (long) valueCountBefore * Integer.BYTES); + int dictSizeAfter = col.getSymbolDictionarySize(); + + if (dictSizeAfter == dictSizeBefore) { + return NativeBufferWriter.varintSize(idx); + } + + long delta = 0; + CharSequence value = col.getSymbolValue(idx); + int utf8Len = utf8Length(value); + delta += NativeBufferWriter.varintSize(utf8Len) + utf8Len; + delta += NativeBufferWriter.varintSize(dictSizeAfter) - NativeBufferWriter.varintSize(dictSizeBefore); + + if (dictSizeBefore > 0 && valueCountBefore > 0) { + int oldMax = dictSizeBefore - 1; + int newMax = dictSizeAfter - 1; + delta += (long) valueCountBefore * ( + NativeBufferWriter.varintSize(newMax) - NativeBufferWriter.varintSize(oldMax) + ); + } + + delta += NativeBufferWriter.varintSize(dictSizeAfter - 1); + return delta; + } + + private static long nonNullablePaddingCost(byte type, int valuesBefore, int missing) { + switch (type) { + case TYPE_BOOLEAN: + return packedBytes(valuesBefore + missing) - packedBytes(valuesBefore); + case TYPE_BYTE: + return missing; + case TYPE_SHORT: + case TYPE_CHAR: + return (long) missing * 2; + case TYPE_INT: + case TYPE_FLOAT: + return (long) missing * 4; + case TYPE_LONG: + case TYPE_DOUBLE: + case TYPE_DATE: + case TYPE_TIMESTAMP: + case TYPE_TIMESTAMP_NANOS: + return (long) missing * 8; + case TYPE_UUID: + return (long) missing * 16; + case TYPE_LONG256: + return (long) missing * 32; + case TYPE_DECIMAL64: + return (long) missing * 8; + case TYPE_DECIMAL128: + return (long) missing * 16; + case TYPE_DECIMAL256: + return (long) missing * 32; + case TYPE_STRING: + case TYPE_VARCHAR: + return (long) missing * 4; + case TYPE_DOUBLE_ARRAY: + case TYPE_LONG_ARRAY: + return (long) missing * 5; + case TYPE_SYMBOL: + throw new IllegalStateException("symbol columns must be nullable"); + default: + return 0; + } + } + + private static int packedBytes(int valueCount) { + return (valueCount + 7) / 8; + } + + private static int utf8Length(CharSequence s) { + if (s == null) { + return 0; + } + int len = 0; + for (int i = 0, n = s.length(); i < n; i++) { + char c = s.charAt(i); + if (c < 0x80) { + len++; + } else if (c < 0x800) { + len += 2; + } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n && Character.isLowSurrogate(s.charAt(i + 1))) { + i++; + len += 4; + } else if (Character.isSurrogate(c)) { + len++; + } else { + len += 3; + } + } + return len; + } + + private QwpTableBuffer.ColumnBuffer acquireColumn(CharSequence name, byte type, boolean useNullBitmap) { + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getExistingColumn(name, type); + if (col == null && currentTableBuffer.getRowCount() > 0) { + // schema change while having some rows accumulated -> we flush committed rows of the current table + // and start from scratch with the new schema + + if (hasInProgressRow()) { + flushCommittedPrefixPreservingCurrentRow(); + } else { + flushCommittedRowsOfCurrentTable(); + } + col = currentTableBuffer.getExistingColumn(name, type); + } + + if (col == null) { + col = currentTableBuffer.getOrCreateColumn(name, type, useNullBitmap); + } + return col; + } + + private QwpTableBuffer.ColumnBuffer acquireDesignatedTimestampColumn(byte type) { + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getExistingColumn("", type); + if (col == null && currentTableBuffer.getRowCount() > 0) { + if (hasInProgressRow()) { + flushCommittedPrefixPreservingCurrentRow(); + } else { + flushCommittedRowsOfCurrentTable(); + } + col = currentTableBuffer.getExistingColumn("", type); + } + if (col == null) { + col = currentTableBuffer.getOrCreateDesignatedTimestampColumn(type); + } + return col; + } + + private void appendDoubleArrayValue(QwpTableBuffer.ColumnBuffer column, Object value) { + if (value instanceof double[]) { + column.addDoubleArray((double[]) value); + return; + } + if (value instanceof double[][]) { + column.addDoubleArray((double[][]) value); + return; + } + if (value instanceof double[][][]) { + column.addDoubleArray((double[][][]) value); + return; + } + if (value instanceof DoubleArray) { + column.addDoubleArray((DoubleArray) value); + return; + } + throw new LineSenderException("unsupported double array type"); + } + + private void appendInProgressColumnState(QwpTableBuffer.ColumnBuffer column) { + ensureInProgressColumnCapacity(inProgressColumnCount + 1); + InProgressColumnState state = inProgressColumns[inProgressColumnCount]; + if (state == null) { + state = new InProgressColumnState(); + inProgressColumns[inProgressColumnCount] = state; + } + state.of(column); + inProgressColumnCount++; + } + + private void appendLongArrayValue(QwpTableBuffer.ColumnBuffer column, Object value) { + if (value instanceof long[]) { + column.addLongArray((long[]) value); + return; + } + if (value instanceof long[][]) { + column.addLongArray((long[][]) value); + return; + } + if (value instanceof long[][][]) { + column.addLongArray((long[][][]) value); + return; + } + if (value instanceof LongArray) { + column.addLongArray((LongArray) value); + return; + } + throw new LineSenderException("unsupported long array type"); + } + + private void atMicros(long timestampMicros) { + try { + stageDesignatedTimestampValue(timestampMicros, false); + commitCurrentRow(); + } catch (RuntimeException | Error e) { + rollbackCurrentRowToCommittedState(); + throw e; + } + } + + private void atNanos(long timestampNanos) { + try { + stageDesignatedTimestampValue(timestampNanos, true); + commitCurrentRow(); + } catch (RuntimeException | Error e) { + rollbackCurrentRowToCommittedState(); + throw e; + } + } + + private void beginColumnWrite(QwpTableBuffer.ColumnBuffer column, CharSequence columnName) { + int columnIndex = column.getIndex(); + ensureStagedColumnMarkCapacity(columnIndex + 1); + if (stagedColumnMarks[columnIndex] == currentRowMark) { + if (columnName != null && columnName.length() == 0) { + throw new LineSenderException("designated timestamp already set for current row"); + } + throw new LineSenderException("column '" + columnName + "' already set for current row"); + } + stagedColumnMarks[columnIndex] = currentRowMark; + appendInProgressColumnState(column); + } + + private void captureInProgressColumnPrefixState() { + int columnCount = currentTableBuffer.getColumnCount(); + ensurePrefixColumnCapacity(columnCount); + for (int i = 0; i < columnCount; i++) { + prefixSizeBefore[i] = -1; + prefixValueCountBefore[i] = -1; + } + for (int i = 0; i < inProgressColumnCount; i++) { + InProgressColumnState state = inProgressColumns[i]; + int columnIndex = state.column.getIndex(); + prefixSizeBefore[columnIndex] = state.sizeBefore; + prefixValueCountBefore[columnIndex] = state.valueCountBefore; + prefixStringDataSizeBefore[columnIndex] = state.stringDataSizeBefore; + prefixArrayShapeOffsetBefore[columnIndex] = state.arrayShapeOffsetBefore; + prefixArrayDataOffsetBefore[columnIndex] = state.arrayDataOffsetBefore; + prefixSymbolDictionarySizeBefore[columnIndex] = state.symbolDictionarySizeBefore; + } + } + + private void checkNotClosed() { + if (closed) { + throw new LineSenderException("Sender is closed"); + } + } + + private void checkTableSelected() { + if (currentTableBuffer == null) { + throw new LineSenderException("table() must be called before adding columns"); + } + } + + private void clearCachedTimestampColumns() { + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; + } + + private void clearInProgressRow() { + inProgressRowValueCount = 0; + for (int i = 0; i < inProgressColumnCount; i++) { + InProgressColumnState state = inProgressColumns[i]; + if (state != null) { + state.clear(); + } + } + if (inProgressColumnCount > 0) { + currentRowMark++; + if (currentRowMark == 0) { + Arrays.fill(stagedColumnMarks, 0); + currentRowMark = 1; + } + } + inProgressColumnCount = 0; + } + + private void clearTransientRowState() { + clearCachedTimestampColumns(); + clearInProgressRow(); + } + + private void commitCurrentRow() { + long estimate = 0; + long committedEstimateBeforeRow = 0; + int targetRows = currentTableBuffer.getRowCount() + 1; + if (trackDatagramEstimate) { + committedEstimateBeforeRow = currentTableBuffer.getRowCount() > 0 + ? committedDatagramEstimate + : estimateBaseForCurrentSchema(); + estimate = estimateCurrentDatagramSizeWithInProgressRow(targetRows); + if (estimate > maxDatagramSize) { + if (currentTableBuffer.getRowCount() == 0) { + throw singleRowTooLarge(estimate); + } + flushCommittedPrefixPreservingCurrentRow(); + targetRows = currentTableBuffer.getRowCount() + 1; + committedEstimateBeforeRow = estimateBaseForCurrentSchema(); + estimate = estimateCurrentDatagramSizeWithInProgressRow(targetRows); + if (estimate > maxDatagramSize) { + throw singleRowTooLarge(estimate); + } + } + } + + currentTableBuffer.nextRow(); + if (trackDatagramEstimate) { + committedDatagramEstimate = estimate; + } + clearInProgressRow(); + if (trackDatagramEstimate) { + long rowDatagramGrowth = estimate - committedEstimateBeforeRow; + recordCommittedRowAndMaybeFlush(rowDatagramGrowth); + } + } + + private void completeColumnWrite() { + inProgressColumns[inProgressColumnCount - 1].captureAfterWrite(); + } + + private int encodeCommittedPrefixPayloadForUdp(QwpTableBuffer tableBuffer) { + payloadWriter.reset(); + columnWriter.setBuffer(payloadWriter); + columnWriter.encodeTable( + tableBuffer, + tableBuffer.getRowCount(), + prefixValueCountBefore, + prefixStringDataSizeBefore, + prefixSymbolDictionarySizeBefore, + false, + false, + false + ); + payloadWriter.finish(); + return payloadWriter.getPosition(); + } + + private int encodeTablePayloadForUdp(QwpTableBuffer tableBuffer) { + payloadWriter.reset(); + columnWriter.setBuffer(payloadWriter); + columnWriter.encodeTable(tableBuffer, false, false, false); + payloadWriter.finish(); + return payloadWriter.getPosition(); + } + + private void ensureInProgressColumnCapacity(int required) { + if (required <= inProgressColumns.length) { + return; + } + + int newCapacity = inProgressColumns.length; + while (newCapacity < required) { + newCapacity *= 2; + } + + InProgressColumnState[] newArr = new InProgressColumnState[newCapacity]; + System.arraycopy(inProgressColumns, 0, newArr, 0, inProgressColumnCount); + inProgressColumns = newArr; + } + + private void ensureNoInProgressRow(String operation) { + if (hasInProgressRow()) { + throw new LineSenderException( + "Cannot " + operation + " while row is in progress. " + + "Use sender.at(), sender.atNow(), or sender.cancelRow() first." + ); + } + } + + private void ensurePrefixColumnCapacity(int required) { + if (required <= prefixSizeBefore.length) { + return; + } + + int newCapacity = prefixSizeBefore.length; + while (newCapacity < required) { + newCapacity *= 2; + } + + prefixSizeBefore = new int[newCapacity]; + prefixValueCountBefore = new int[newCapacity]; + prefixStringDataSizeBefore = new long[newCapacity]; + prefixArrayShapeOffsetBefore = new int[newCapacity]; + prefixArrayDataOffsetBefore = new int[newCapacity]; + prefixSymbolDictionarySizeBefore = new int[newCapacity]; + } + + private void ensureStagedColumnMarkCapacity(int required) { + if (required <= stagedColumnMarks.length) { + return; + } + + int newCapacity = stagedColumnMarks.length; + while (newCapacity < required) { + newCapacity *= 2; + } + + int[] newArr = new int[newCapacity]; + System.arraycopy(stagedColumnMarks, 0, newArr, 0, stagedColumnMarks.length); + stagedColumnMarks = newArr; + } + + private long estimateBaseForCurrentSchema() { + if (currentTableHeadroomState != null) { + long cachedEstimate = currentTableHeadroomState.getCachedBaseEstimate(currentTableBuffer.getColumnCount()); + if (cachedEstimate > -1) { + return cachedEstimate; + } + } + + long estimate = HEADER_SIZE; + int tableNameUtf8 = NativeBufferWriter.utf8Length(currentTableName); + estimate += NativeBufferWriter.varintSize(tableNameUtf8) + tableNameUtf8; + estimate += VARINT_INT_UPPER_BOUND; + estimate += VARINT_INT_UPPER_BOUND; + estimate += 1; + + QwpColumnDef[] defs = currentTableBuffer.getColumnDefs(); + for (QwpColumnDef def : defs) { + int nameUtf8 = NativeBufferWriter.utf8Length(def.getName()); + estimate += NativeBufferWriter.varintSize(nameUtf8) + nameUtf8; + estimate += 1; + + byte type = def.getTypeCode(); + if (type == TYPE_STRING || type == TYPE_VARCHAR) { + estimate += 4; + } else if (type == TYPE_SYMBOL) { + estimate += 1; + } else if (type == TYPE_DECIMAL64 || type == TYPE_DECIMAL128 || type == TYPE_DECIMAL256) { + estimate += 1; + } + } + if (currentTableHeadroomState != null) { + currentTableHeadroomState.cacheBaseEstimate(currentTableBuffer.getColumnCount(), estimate); + } + return estimate; + } + + private long estimateCurrentDatagramSizeWithInProgressRow(int targetRows) { + long estimate = currentTableBuffer.getRowCount() > 0 ? committedDatagramEstimate : estimateBaseForCurrentSchema(); + for (int i = 0; i < inProgressColumnCount; i++) { + InProgressColumnState state = inProgressColumns[i]; + estimate += state.payloadEstimateDelta; + if (state.useNullBitmap) { + estimate += bitmapBytes(targetRows) - bitmapBytes(state.sizeBefore); + } + } + ensureStagedColumnMarkCapacity(currentTableBuffer.getColumnCount()); + for (int i = 0, columnCount = currentTableBuffer.getColumnCount(); i < columnCount; i++) { + if (stagedColumnMarks[i] == currentRowMark) { + continue; + } + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getColumn(i); + int missing = targetRows - col.getSize(); + if (col.usesNullBitmap()) { + estimate += bitmapBytes(targetRows) - bitmapBytes(col.getSize()); + } else { + estimate += nonNullablePaddingCost(col.getType(), col.getValueCount(), missing); + } + } + return estimate; + } + + private void flushCommittedPrefixPreservingCurrentRow() { + if (currentTableBuffer == null || currentTableBuffer.getRowCount() == 0) { + return; + } + + captureInProgressColumnPrefixState(); + sendCommittedPrefix(currentTableName, currentTableBuffer); + currentTableBuffer.retainInProgressRow( + prefixSizeBefore, + prefixValueCountBefore, + prefixArrayShapeOffsetBefore, + prefixArrayDataOffsetBefore + ); + resetCommittedDatagramEstimate(); + for (int i = 0; i < inProgressColumnCount; i++) { + inProgressColumns[i].rebaseToEmptyTable(); + } + } + + private void flushCommittedRowsOfCurrentTable() { + if (currentTableBuffer == null || currentTableBuffer.getRowCount() == 0) { + return; + } + sendWholeTableBuffer(currentTableName, currentTableBuffer); + clearCachedTimestampColumns(); + resetCommittedDatagramEstimate(); + } + + private void flushInternal() { + ObjList keys = tableBuffers.keys(); + for (int i = 0, n = keys.size(); i < n; i++) { + CharSequence tableName = keys.getQuick(i); + if (tableName == null) { + continue; + } + QwpTableBuffer tableBuffer = tableBuffers.get(tableName); + if (tableBuffer == null || tableBuffer.getRowCount() == 0) { + continue; + } + sendWholeTableBuffer(tableName, tableBuffer); + } + clearTransientRowState(); + resetCommittedDatagramEstimate(); + } + + private void flushSingleTable(String tableName, QwpTableBuffer tableBuffer) { + sendWholeTableBuffer(tableName, tableBuffer); + clearTransientRowState(); + resetCommittedDatagramEstimate(); + } + + private boolean hasInProgressRow() { + return inProgressColumnCount > 0; + } + + private void recordCommittedRowAndMaybeFlush(long rowDatagramGrowth) { + if (currentTableBuffer == null || currentTableBuffer.getRowCount() == 0 || currentTableHeadroomState == null) { + return; + } + + currentTableHeadroomState.recordCommittedRow(currentTableBuffer.getColumnCount(), rowDatagramGrowth); + if (shouldFlushCommittedRowsAfterCommit()) { + flushCommittedRowsOfCurrentTable(); + } + } + + private void resetCommittedDatagramEstimate() { + committedDatagramEstimate = 0; + } + + private void rollbackCurrentRowToCommittedState() { + if (currentTableBuffer != null) { + currentTableBuffer.cancelCurrentRow(); + currentTableBuffer.rollbackUncommittedColumns(); + } + clearCachedTimestampColumns(); + clearInProgressRow(); + } + + private void sendCommittedPrefix(CharSequence tableName, QwpTableBuffer tableBuffer) { + int payloadLength = encodeCommittedPrefixPayloadForUdp(tableBuffer); + sendEncodedPayload(tableName, payloadLength); + } + + private void sendEncodedPayload(CharSequence tableName, int payloadLength) { + headerBuffer.reset(); + headerBuffer.putByte((byte) 'Q'); + headerBuffer.putByte((byte) 'W'); + headerBuffer.putByte((byte) 'P'); + headerBuffer.putByte((byte) '1'); + headerBuffer.putByte(VERSION_1); + headerBuffer.putByte((byte) 0); + headerBuffer.putShort((short) 1); + headerBuffer.putInt(payloadLength); + + datagramSegments.reset(); + datagramSegments.add(headerBuffer.getBufferPtr(), headerBuffer.getPosition()); + datagramSegments.appendFrom(payloadWriter.getSegments()); + + try { + channel.sendSegments( + datagramSegments.getAddress(), + datagramSegments.getSegmentCount(), + (int) datagramSegments.getTotalLength() + ); + } catch (LineSenderException e) { + LOG.warn("UDP send failed [table={}, errno={}]: {}", tableName, channel.errno(), String.valueOf(e)); + } + } + + private void sendWholeTableBuffer(CharSequence tableName, QwpTableBuffer tableBuffer) { + int payloadLength = encodeTablePayloadForUdp(tableBuffer); + sendEncodedPayload(tableName, payloadLength); + tableBuffer.reset(); + } + + private boolean shouldFlushCommittedRowsAfterCommit() { + if (committedDatagramEstimate >= maxDatagramSize) { + return true; + } + + long predictedNextRowGrowth = currentTableHeadroomState.predictNextRowGrowth(); + if (predictedNextRowGrowth <= 0) { + return false; + } + return committedDatagramEstimate > maxDatagramSize - predictedNextRowGrowth; + } + + private LineSenderException singleRowTooLarge(long estimate) { + return new LineSenderException( + "single row exceeds maximum datagram size (" + maxDatagramSize + + " bytes), estimated " + estimate + " bytes" + ); + } + + private void stageBooleanColumnValue(CharSequence name, boolean value) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_BOOLEAN, false); + beginColumnWrite(col, name); + col.addBoolean(value); + completeColumnWrite(); + inProgressRowValueCount++; + } + + private void stageDecimal128ColumnValue(CharSequence name, Decimal128 value) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DECIMAL128, true); + beginColumnWrite(col, name); + col.addDecimal128(value); + completeColumnWrite(); + inProgressRowValueCount++; + } + + private void stageDecimal256ColumnValue(CharSequence name, Decimal256 value) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DECIMAL256, true); + beginColumnWrite(col, name); + col.addDecimal256(value); + completeColumnWrite(); + inProgressRowValueCount++; + } + + private void stageDecimal64ColumnValue(CharSequence name, Decimal64 value) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DECIMAL64, true); + beginColumnWrite(col, name); + col.addDecimal64(value); + completeColumnWrite(); + inProgressRowValueCount++; + } + + private void stageDesignatedTimestampValue(long value, boolean nanos) { + QwpTableBuffer.ColumnBuffer col; + byte type = nanos ? TYPE_TIMESTAMP_NANOS : TYPE_TIMESTAMP; + col = nanos ? cachedTimestampNanosColumn : cachedTimestampColumn; + if (col == null) { + col = acquireDesignatedTimestampColumn(type); + if (nanos) { + cachedTimestampNanosColumn = col; + } else { + cachedTimestampColumn = col; + } + } + beginColumnWrite(col, ""); + col.addLong(value); + completeColumnWrite(); + } + + private void stageDoubleArrayColumnValue(CharSequence name, Object value) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DOUBLE_ARRAY, true); + beginColumnWrite(col, name); + appendDoubleArrayValue(col, value); + completeColumnWrite(); + inProgressRowValueCount++; + } + + private void stageDoubleColumnValue(CharSequence name, double value) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DOUBLE, false); + beginColumnWrite(col, name); + col.addDouble(value); + completeColumnWrite(); + inProgressRowValueCount++; + } + + private void stageLongArrayColumnValue(CharSequence name, Object value) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_LONG_ARRAY, true); + beginColumnWrite(col, name); + appendLongArrayValue(col, value); + completeColumnWrite(); + inProgressRowValueCount++; + } + + private void stageLongColumnValue(CharSequence name, long value) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_LONG, false); + beginColumnWrite(col, name); + col.addLong(value); + completeColumnWrite(); + inProgressRowValueCount++; + } + + private void stageNullArrayColumnValue(CharSequence name, byte type) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, type, true); + beginColumnWrite(col, name); + col.addNull(); + completeColumnWrite(); + inProgressRowValueCount++; + } + + private void stageStringColumnValue(CharSequence name, CharSequence value) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_STRING, true); + beginColumnWrite(col, name); + col.addString(value); + completeColumnWrite(); + inProgressRowValueCount++; + } + + private void stageSymbolColumnValue(CharSequence name, CharSequence value) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_SYMBOL, true); + beginColumnWrite(col, name); + col.addSymbol(value); + completeColumnWrite(); + inProgressRowValueCount++; + } + + private void stageTimestampColumnValue(CharSequence name, byte type, long value) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, type, true); + beginColumnWrite(col, name); + col.addLong(value); + completeColumnWrite(); + inProgressRowValueCount++; + } + + private long toMicros(long value, ChronoUnit unit) { + switch (unit) { + case NANOS: + return value / 1000L; + case MICROS: + return value; + case MILLIS: + return value * 1000L; + case SECONDS: + return value * 1_000_000L; + case MINUTES: + return value * 60_000_000L; + case HOURS: + return value * 3_600_000_000L; + case DAYS: + return value * 86_400_000_000L; + default: + throw new LineSenderException("Unsupported time unit: " + unit); + } + } + + /** + * Captures the state of a column buffer at the moment the in-progress row starts + * writing to it. The snapshot allows the sender to compute incremental datagram + * size estimates and to roll back the column to its pre-row state on error or cancel. + */ + private static final class InProgressColumnState { + private int arrayDataOffsetBefore; + private int arrayShapeOffsetBefore; + private QwpTableBuffer.ColumnBuffer column; + private boolean useNullBitmap; + private long payloadEstimateDelta; + private int sizeBefore; + private long stringDataSizeBefore; + private int symbolDictionarySizeBefore; + private int valueCountBefore; + + void captureAfterWrite() { + this.payloadEstimateDelta = estimateInProgressColumnPayload(this); + } + + void clear() { + column = null; + useNullBitmap = false; + payloadEstimateDelta = 0; + } + + void of(QwpTableBuffer.ColumnBuffer column) { + this.column = column; + this.useNullBitmap = column.usesNullBitmap(); + this.payloadEstimateDelta = 0; + this.sizeBefore = column.getSize(); + this.valueCountBefore = column.getValueCount(); + this.stringDataSizeBefore = column.getStringDataSize(); + this.arrayShapeOffsetBefore = column.getArrayShapeOffset(); + this.arrayDataOffsetBefore = column.getArrayDataOffset(); + this.symbolDictionarySizeBefore = column.getSymbolDictionarySize(); + } + + void rebaseToEmptyTable() { + this.sizeBefore = 0; + this.valueCountBefore = 0; + this.stringDataSizeBefore = 0; + this.arrayShapeOffsetBefore = 0; + this.arrayDataOffsetBefore = 0; + this.symbolDictionarySizeBefore = 0; + captureAfterWrite(); + } + } + + private static final class TableHeadroomState { + private long cachedBaseEstimate = -1; + private int cachedBaseEstimateColumnCount = -1; + private int committedSampleCount; + private long ewmaRowDatagramGrowth; + private long lastRowDatagramGrowth; + private int schemaColumnCount = -1; + + void cacheBaseEstimate(int currentSchemaColumnCount, long estimate) { + cachedBaseEstimateColumnCount = currentSchemaColumnCount; + cachedBaseEstimate = estimate; + } + + long getCachedBaseEstimate(int currentSchemaColumnCount) { + return cachedBaseEstimateColumnCount == currentSchemaColumnCount ? cachedBaseEstimate : -1; + } + + long predictNextRowGrowth() { + if (committedSampleCount < 2) { + return 0; + } + return Math.max(lastRowDatagramGrowth, ewmaRowDatagramGrowth); + } + + void recordCommittedRow(int currentSchemaColumnCount, long rowDatagramGrowth) { + if (schemaColumnCount != currentSchemaColumnCount) { + committedSampleCount = 0; + ewmaRowDatagramGrowth = 0; + lastRowDatagramGrowth = 0; + schemaColumnCount = currentSchemaColumnCount; + } + + lastRowDatagramGrowth = rowDatagramGrowth; + if (committedSampleCount == 0) { + ewmaRowDatagramGrowth = rowDatagramGrowth; + } else { + ewmaRowDatagramGrowth += (rowDatagramGrowth - ewmaRowDatagramGrowth) >> ADAPTIVE_HEADROOM_EWMA_SHIFT; + } + if (committedSampleCount < Integer.MAX_VALUE) { + committedSampleCount++; + } + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java new file mode 100644 index 0000000..56098cf --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -0,0 +1,127 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.QuietCloseable; + +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; + +/** + * Encodes QWP v1 messages for WebSocket transport. + *

    + * This encoder delegates column encoding to {@link QwpColumnWriter} and wraps + * the encoded payload with a 12-byte ILP4 header. + */ +public class QwpWebSocketEncoder implements QuietCloseable { + + private final QwpColumnWriter columnWriter = new QwpColumnWriter(); + private NativeBufferWriter buffer; + private byte flags; + + public QwpWebSocketEncoder() { + this.buffer = new NativeBufferWriter(); + this.flags = 0; + } + + public QwpWebSocketEncoder(int bufferSize) { + this.buffer = new NativeBufferWriter(bufferSize); + this.flags = 0; + } + + @Override + public void close() { + if (buffer != null) { + buffer.close(); + buffer = null; + } + } + + public int encode(QwpTableBuffer tableBuffer, boolean useSchemaRef) { + buffer.reset(); + writeHeader(1, 0); + int payloadStart = buffer.getPosition(); + columnWriter.setBuffer(buffer); + columnWriter.encodeTable(tableBuffer, useSchemaRef, false, isGorillaEnabled()); + int payloadLength = buffer.getPosition() - payloadStart; + buffer.patchInt(8, payloadLength); + return buffer.getPosition(); + } + + public int encodeWithDeltaDict( + QwpTableBuffer tableBuffer, + GlobalSymbolDictionary globalDict, + int confirmedMaxId, + int batchMaxId, + boolean useSchemaRef + ) { + buffer.reset(); + int deltaStart = confirmedMaxId + 1; + int deltaCount = Math.max(0, batchMaxId - confirmedMaxId); + byte savedFlags = flags; + flags |= FLAG_DELTA_SYMBOL_DICT; + writeHeader(1, 0); + int payloadStart = buffer.getPosition(); + buffer.putVarint(deltaStart); + buffer.putVarint(deltaCount); + for (int id = deltaStart; id < deltaStart + deltaCount; id++) { + String symbol = globalDict.getSymbol(id); + buffer.putString(symbol); + } + columnWriter.setBuffer(buffer); + columnWriter.encodeTable(tableBuffer, useSchemaRef, true, isGorillaEnabled()); + int payloadLength = buffer.getPosition() - payloadStart; + buffer.patchInt(8, payloadLength); + flags = savedFlags; + return buffer.getPosition(); + } + + public QwpBufferWriter getBuffer() { + return buffer; + } + + public boolean isGorillaEnabled() { + return (flags & FLAG_GORILLA) != 0; + } + + public void setGorillaEnabled(boolean enabled) { + if (enabled) { + flags |= FLAG_GORILLA; + } else { + flags &= ~FLAG_GORILLA; + } + } + + public void writeHeader(int tableCount, int payloadLength) { + buffer.putByte((byte) 'Q'); + buffer.putByte((byte) 'W'); + buffer.putByte((byte) 'P'); + buffer.putByte((byte) '1'); + buffer.putByte(VERSION_1); + buffer.putByte(flags); + buffer.putShort((short) tableCount); + buffer.putInt(payloadLength); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java new file mode 100644 index 0000000..92e19d3 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -0,0 +1,1462 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.Sender; +import io.questdb.client.cairo.TableUtils; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketClientFactory; +import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.line.array.DoubleArray; +import io.questdb.client.cutlass.line.array.LongArray; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.CharSequenceObjHashMap; +import io.questdb.client.std.Chars; +import io.questdb.client.std.Decimal128; +import io.questdb.client.std.Decimal256; +import io.questdb.client.std.Decimal64; +import io.questdb.client.std.LongHashSet; +import io.questdb.client.std.ObjList; +import io.questdb.client.std.bytes.DirectByteSlice; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.TestOnly; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.concurrent.TimeUnit; + +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; + + +/** + * QWP v1 WebSocket client sender for streaming data to QuestDB. + *

    + * This sender uses a double-buffering scheme with asynchronous I/O for high throughput: + *

      + *
    • User thread writes rows to the active microbatch buffer
      • + *
    • When buffer is full (row count, byte size, or age), it's sealed and enqueued
      • + *
    • A dedicated I/O thread sends batches asynchronously
      • + *
    • Double-buffering ensures one buffer is always available for writing
      • + *
    + *

    + * Configuration options: + *

      + *
    • {@code autoFlushRows} - Maximum rows per batch (default: 1000)
      • + *
    • {@code autoFlushBytes} - Maximum bytes per batch (default: 128KB)
      • + *
    • {@code autoFlushIntervalNanos} - Maximum age before auto-flush (default: 100ms)
      • + *
    + *

    + * Example usage: + * 

    + * try (QwpWebSocketSender sender = QwpWebSocketSender.connect("localhost", 9000)) {
+ *     for (int i = 0; i < 100_000; i++) {
+ *         sender.table("metrics")
+ *               .symbol("host", "server-" + (i % 10))
+ *               .doubleColumn("cpu", Math.random() * 100)
+ *               .atNow();
+ *         // Rows are batched and sent asynchronously!
+ *     }
+ *     // flush() waits for all pending batches to be sent
+ *     sender.flush();
+ * }
+ *
    + */ +public class QwpWebSocketSender implements Sender { + + public static final int DEFAULT_AUTO_FLUSH_BYTES = 128 * 1024; // 128KB + public static final long DEFAULT_AUTO_FLUSH_INTERVAL_NANOS = 100_000_000L; // 100ms + public static final int DEFAULT_AUTO_FLUSH_ROWS = 1_000; + public static final int DEFAULT_IN_FLIGHT_WINDOW_SIZE = 128; + private static final int DEFAULT_BUFFER_SIZE = 8192; + private static final int DEFAULT_MICROBATCH_BUFFER_SIZE = 1024 * 1024; // 1MB + private static final Logger LOG = LoggerFactory.getLogger(QwpWebSocketSender.class); + private static final int MAX_TABLE_NAME_LENGTH = 127; + private static final String WRITE_PATH = "/write/v4"; + private final AckFrameHandler ackHandler = new AckFrameHandler(this); + private final WebSocketResponse ackResponse = new WebSocketResponse(); + private final String authorizationHeader; + private final int autoFlushBytes; + private final long autoFlushIntervalNanos; + // Auto-flush configuration + private final int autoFlushRows; + private final Decimal256 currentDecimal256 = new Decimal256(); + // Encoder for QWP v1 messages + private final QwpWebSocketEncoder encoder; + // Global symbol dictionary for delta encoding + private final GlobalSymbolDictionary globalSymbolDictionary; + private final String host; + // Flow control configuration + private final int inFlightWindowSize; + private final int port; + // Track schema hashes that have been sent to the server (for schema reference mode) + // First time we send a schema: full schema. Subsequent times: 8-byte hash reference. + // Combined key = schemaHash XOR (tableNameHash << 32) to include table name in lookup. + private final LongHashSet sentSchemaHashes = new LongHashSet(); + private final CharSequenceObjHashMap tableBuffers; + private final boolean tlsEnabled; + private MicrobatchBuffer activeBuffer; + // Double-buffering for async I/O + private MicrobatchBuffer buffer0; + private MicrobatchBuffer buffer1; + // Cached column references to avoid repeated hashmap lookups + private QwpTableBuffer.ColumnBuffer cachedTimestampColumn; + private QwpTableBuffer.ColumnBuffer cachedTimestampNanosColumn; + // WebSocket client (zero-GC native implementation) + private WebSocketClient client; + private boolean closed; + private boolean connected; + // Track max global symbol ID used in current batch (for delta calculation) + private int currentBatchMaxSymbolId = -1; + private QwpTableBuffer currentTableBuffer; + private String currentTableName; + private long firstPendingRowTimeNanos; + // Configuration + private boolean gorillaEnabled = true; + // Flow control + private InFlightWindow inFlightWindow; + // Track highest symbol ID sent to server (for delta encoding) + // Once sent over TCP, server is guaranteed to receive it (or connection dies) + private volatile int maxSentSymbolId = -1; + // Batch sequence counter (must match server's messageSequence) + private long nextBatchSequence = 0; + // Async mode: pending row tracking + private long pendingBytes; + private int pendingRowCount; + private boolean sawBinaryAck; + private WebSocketSendQueue sendQueue; + + private QwpWebSocketSender( + String host, + int port, + boolean tlsEnabled, + int bufferSize, + int autoFlushRows, + int autoFlushBytes, + long autoFlushIntervalNanos, + int inFlightWindowSize, + String authorizationHeader + ) { + this.authorizationHeader = authorizationHeader; + this.host = host; + this.port = port; + this.tlsEnabled = tlsEnabled; + this.encoder = new QwpWebSocketEncoder(bufferSize); + this.tableBuffers = new CharSequenceObjHashMap<>(); + this.currentTableBuffer = null; + this.currentTableName = null; + this.connected = false; + this.closed = false; + this.autoFlushRows = autoFlushRows; + this.autoFlushBytes = autoFlushBytes; + this.autoFlushIntervalNanos = autoFlushIntervalNanos; + this.inFlightWindowSize = inFlightWindowSize; + + // Initialize global symbol dictionary for delta encoding + this.globalSymbolDictionary = new GlobalSymbolDictionary(); + + // Initialize double-buffering if async mode (window > 1) + if (inFlightWindowSize > 1) { + int microbatchBufferSize = Math.max(DEFAULT_MICROBATCH_BUFFER_SIZE, autoFlushBytes * 2); + try { + this.buffer0 = new MicrobatchBuffer(microbatchBufferSize, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos); + this.buffer1 = new MicrobatchBuffer(microbatchBufferSize, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos); + } catch (Throwable t) { + if (buffer0 != null) { + buffer0.close(); + } + encoder.close(); + throw t; + } + this.activeBuffer = buffer0; + } + } + + /** + * Creates a new sender and connects to the specified host and port. + * Uses default auto-flush settings and in-flight window size. + * + * @param host server host + * @param port server HTTP port (WebSocket upgrade happens on same port) + * @return connected sender + */ + public static QwpWebSocketSender connect(String host, int port) { + return connect(host, port, false); + } + + /** + * Creates a new sender and connects to the specified host and port. + * Uses default auto-flush settings and in-flight window size. + * + * @param host server host + * @param port server HTTP port + * @param tlsEnabled whether to use TLS + * @return connected sender + */ + public static QwpWebSocketSender connect(String host, int port, boolean tlsEnabled) { + return connect( + host, port, tlsEnabled, + DEFAULT_AUTO_FLUSH_ROWS, DEFAULT_AUTO_FLUSH_BYTES, DEFAULT_AUTO_FLUSH_INTERVAL_NANOS, + DEFAULT_IN_FLIGHT_WINDOW_SIZE, null + ); + } + + /** + * Creates a new sender with full configuration and connects. + *

    + * In-flight window size controls the flow behavior: 1 means synchronous (each batch + * waits for ACK), greater than 1 enables asynchronous pipelining with a background I/O thread. + * + * @param host server host + * @param port server HTTP port + * @param tlsEnabled whether to use TLS + * @param autoFlushRows rows per batch (0 = no limit) + * @param autoFlushBytes bytes per batch (0 = no limit) + * @param autoFlushIntervalNanos age before flush in nanos (0 = no limit) + * @param inFlightWindowSize max batches awaiting server ACK (1 = sync, default: 128) + * @param authorizationHeader HTTP Authorization header value, or null + * @return connected sender + */ + public static QwpWebSocketSender connect( + String host, + int port, + boolean tlsEnabled, + int autoFlushRows, + int autoFlushBytes, + long autoFlushIntervalNanos, + int inFlightWindowSize, + String authorizationHeader + ) { + QwpWebSocketSender sender = new QwpWebSocketSender( + host, port, tlsEnabled, DEFAULT_BUFFER_SIZE, + autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, + inFlightWindowSize, authorizationHeader + ); + try { + sender.ensureConnected(); + } catch (Throwable t) { + sender.close(); + throw t; + } + return sender; + } + + /** + * Creates a sender without connecting. For testing only. + *

    + * This allows unit tests to test sender logic without requiring a real server. + * Uses default auto-flush settings. + * + * @param host server host (not connected) + * @param port server port (not connected) + * @param inFlightWindowSize window size: 1 for sync behavior, >1 for async + * @return unconnected sender + */ + public static QwpWebSocketSender createForTesting(String host, int port, int inFlightWindowSize) { + return new QwpWebSocketSender( + host, port, false, DEFAULT_BUFFER_SIZE, + DEFAULT_AUTO_FLUSH_ROWS, DEFAULT_AUTO_FLUSH_BYTES, DEFAULT_AUTO_FLUSH_INTERVAL_NANOS, + inFlightWindowSize, null + ); + } + + /** + * Creates a sender with custom flow control settings without connecting. For testing only. + * + * @param host server host (not connected) + * @param port server port (not connected) + * @param autoFlushRows rows per batch (0 = no limit) + * @param autoFlushBytes bytes per batch (0 = no limit) + * @param autoFlushIntervalNanos age before flush in nanos (0 = no limit) + * @param inFlightWindowSize window size: 1 for sync behavior, >1 for async + * @return unconnected sender + */ + public static QwpWebSocketSender createForTesting( + String host, + int port, + int autoFlushRows, + int autoFlushBytes, + long autoFlushIntervalNanos, + int inFlightWindowSize + ) { + return new QwpWebSocketSender( + host, port, false, DEFAULT_BUFFER_SIZE, + autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, + inFlightWindowSize, null + ); + } + + @Override + public void at(long timestamp, ChronoUnit unit) { + checkNotClosed(); + checkTableSelected(); + if (unit == ChronoUnit.NANOS) { + atNanos(timestamp); + } else { + long micros = toMicros(timestamp, unit); + atMicros(micros); + } + } + + @Override + public void at(Instant timestamp) { + checkNotClosed(); + checkTableSelected(); + long micros = timestamp.getEpochSecond() * 1_000_000L + timestamp.getNano() / 1000L; + atMicros(micros); + } + + @Override + public void atNow() { + checkNotClosed(); + checkTableSelected(); + // Server-assigned timestamp - just send the row without designated timestamp + sendRow(); + } + + @Override + public QwpWebSocketSender boolColumn(CharSequence columnName, boolean value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_BOOLEAN, false); + if (col != null) { + col.addBoolean(value); + } + return this; + } + + @Override + public DirectByteSlice bufferView() { + throw new LineSenderException("bufferView() is not supported for WebSocket sender"); + } + + /** + * Adds a BYTE column value to the current row. + * + * @param columnName the column name + * @param value the byte value + * @return this sender for method chaining + */ + public QwpWebSocketSender byteColumn(CharSequence columnName, byte value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_BYTE, false); + if (col != null) { + col.addByte(value); + } + return this; + } + + @Override + public void cancelRow() { + checkNotClosed(); + if (currentTableBuffer != null) { + currentTableBuffer.cancelCurrentRow(); + } + } + + /** + * Adds a CHAR column value to the current row. + *

    + * CHAR is stored as a 2-byte UTF-16 code unit in QuestDB. + * + * @param columnName the column name + * @param value the character value + * @return this sender for method chaining + */ + public QwpWebSocketSender charColumn(CharSequence columnName, char value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_CHAR, false); + if (col != null) { + col.addShort((short) value); + } + return this; + } + + @Override + public void close() { + if (!closed) { + closed = true; + + // Flush any remaining data + try { + if (inFlightWindowSize > 1) { + // Async mode (window > 1): flush accumulated rows in table buffers first + flushPendingRows(); + + if (activeBuffer != null && activeBuffer.hasData()) { + sealAndSwapBuffer(); + } + // Wait for all batches to be sent and acknowledged before closing + if (sendQueue != null) { + sendQueue.flush(); + sendQueue.awaitPendingAcks(); + } else if (inFlightWindow != null) { + inFlightWindow.awaitEmpty(); + } + } else { + // Sync mode (window=1): flush pending rows synchronously + if (pendingRowCount > 0 && client != null && client.isConnected()) { + flushSync(); + } + } + } catch (Exception e) { + LOG.error("Error during close: {}", String.valueOf(e)); + } + + // Shut down the I/O thread before closing the socket or buffers + // it may be using. This must run even if the flush above failed. + if (sendQueue != null) { + try { + sendQueue.close(); + } catch (Exception e) { + LOG.error("Error closing send queue: {}", String.valueOf(e)); + } + } + + // Close buffers (async mode only, window > 1) + if (buffer0 != null) { + buffer0.close(); + } + if (buffer1 != null) { + buffer1.close(); + } + + if (client != null) { + client.close(); + client = null; + } + encoder.close(); + // Close all table buffers to free off-heap column memory + ObjList keys = tableBuffers.keys(); + for (int i = 0, n = keys.size(); i < n; i++) { + CharSequence key = keys.getQuick(i); + if (key != null) { + QwpTableBuffer tb = tableBuffers.get(key); + if (tb != null) { + tb.close(); + } + } + } + tableBuffers.clear(); + + LOG.info("QwpWebSocketSender closed"); + } + } + + @Override + public Sender decimalColumn(CharSequence name, Decimal64 value) { + if (value == null || value.isNull()) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DECIMAL64, true); + if (col != null) { + col.addDecimal64(value); + } + return this; + } + + @Override + public Sender decimalColumn(CharSequence name, Decimal128 value) { + if (value == null || value.isNull()) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DECIMAL128, true); + if (col != null) { + col.addDecimal128(value); + } + return this; + } + + @Override + public Sender decimalColumn(CharSequence name, Decimal256 value) { + if (value == null || value.isNull()) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DECIMAL256, true); + if (col != null) { + col.addDecimal256(value); + } + return this; + } + + @Override + public Sender decimalColumn(CharSequence name, CharSequence value) { + if (value == null || value.length() == 0) return this; + checkNotClosed(); + checkTableSelected(); + try { + currentDecimal256.ofString(value); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DECIMAL256, true); + if (col != null) { + col.addDecimal256(currentDecimal256); + } + } catch (Exception e) { + throw new LineSenderException("Failed to parse decimal value: " + value, e); + } + return this; + } + + @Override + public Sender doubleArray(@NotNull CharSequence name, double[] values) { + if (values == null) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DOUBLE_ARRAY, true); + if (col != null) { + col.addDoubleArray(values); + } + return this; + } + + @Override + public Sender doubleArray(@NotNull CharSequence name, double[][] values) { + if (values == null) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DOUBLE_ARRAY, true); + if (col != null) { + col.addDoubleArray(values); + } + return this; + } + + @Override + public Sender doubleArray(@NotNull CharSequence name, double[][][] values) { + if (values == null) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DOUBLE_ARRAY, true); + if (col != null) { + col.addDoubleArray(values); + } + return this; + } + + @Override + public Sender doubleArray(CharSequence name, DoubleArray array) { + if (array == null) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DOUBLE_ARRAY, true); + if (col != null) { + col.addDoubleArray(array); + } + return this; + } + + @Override + public QwpWebSocketSender doubleColumn(CharSequence columnName, double value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_DOUBLE, true); + if (col != null) { + col.addDouble(value); + } + return this; + } + + /** + * Adds a FLOAT column value to the current row. + * + * @param columnName the column name + * @param value the float value + * @return this sender for method chaining + */ + public QwpWebSocketSender floatColumn(CharSequence columnName, float value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_FLOAT, true); + if (col != null) { + col.addFloat(value); + } + return this; + } + + @Override + public void flush() { + checkNotClosed(); + ensureConnected(); + + if (inFlightWindowSize > 1) { + // Async mode (window > 1): flush pending rows and wait for ACKs + flushPendingRows(); + + // Flush any remaining data in the active microbatch buffer + if (activeBuffer.hasData()) { + sealAndSwapBuffer(); + } + + // Wait for all pending batches to be sent to the server + sendQueue.flush(); + + // Wait for all in-flight batches to be acknowledged by the server + if (sendQueue != null) { + sendQueue.awaitPendingAcks(); + } else { + inFlightWindow.awaitEmpty(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Flush complete [totalBatches={}, totalBytes={}, totalAcked={}]", sendQueue.getTotalBatchesSent(), sendQueue.getTotalBytesSent(), inFlightWindow.getTotalAcked()); + } + } else { + // Sync mode (window=1): flush pending rows and wait for ACKs synchronously + flushSync(); + } + } + + /** + * Returns the auto-flush byte threshold. + */ + public int getAutoFlushBytes() { + return autoFlushBytes; + } + + /** + * Returns the auto-flush interval in nanoseconds. + */ + public long getAutoFlushIntervalNanos() { + return autoFlushIntervalNanos; + } + + /** + * Returns the auto-flush row threshold. + */ + public int getAutoFlushRows() { + return autoFlushRows; + } + + /** + * Returns the max symbol ID sent to the server. + * Once sent over TCP, server is guaranteed to receive it (or connection dies). + */ + public int getMaxSentSymbolId() { + return maxSentSymbolId; + } + + /** + * Registers a symbol value in the global dictionary and returns its global ID. + * Called from {@link QwpTableBuffer.ColumnBuffer#addSymbol(CharSequence)}. + * + * @param symbol the symbol value to register + * @return the global symbol ID + */ + public int getOrAddGlobalSymbol(CharSequence symbol) { + int globalId = globalSymbolDictionary.getOrAddSymbol(symbol); + if (globalId > currentBatchMaxSymbolId) { + currentBatchMaxSymbolId = globalId; + } + return globalId; + } + + /** + * Returns the number of pending rows not yet flushed. + * For testing. + */ + public int getPendingRowCount() { + return pendingRowCount; + } + + @TestOnly + public QwpTableBuffer getTableBuffer(String tableName) { + QwpTableBuffer buffer = tableBuffers.get(tableName); + if (buffer == null) { + buffer = new QwpTableBuffer(tableName, this); + tableBuffers.put(tableName, buffer); + } + currentTableBuffer = buffer; + currentTableName = tableName; + return buffer; + } + + /** + * Adds an INT column value to the current row. + * + * @param columnName the column name + * @param value the int value + * @return this sender for method chaining + */ + public QwpWebSocketSender intColumn(CharSequence columnName, int value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_INT, true); + if (col != null) { + col.addInt(value); + } + return this; + } + + /** + * Returns whether Gorilla encoding is enabled. + */ + public boolean isGorillaEnabled() { + return gorillaEnabled; + } + + /** + * Adds a LONG256 column value to the current row. + * + * @param columnName the column name + * @param l0 the least significant 64 bits + * @param l1 the second 64 bits + * @param l2 the third 64 bits + * @param l3 the most significant 64 bits + * @return this sender for method chaining + */ + public QwpWebSocketSender long256Column(CharSequence columnName, long l0, long l1, long l2, long l3) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_LONG256, true); + if (col != null) { + col.addLong256(l0, l1, l2, l3); + } + return this; + } + + @Override + public Sender longArray(@NotNull CharSequence name, long[] values) { + if (values == null) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_LONG_ARRAY, true); + if (col != null) { + col.addLongArray(values); + } + return this; + } + + @Override + public Sender longArray(@NotNull CharSequence name, long[][] values) { + if (values == null) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_LONG_ARRAY, true); + if (col != null) { + col.addLongArray(values); + } + return this; + } + + @Override + public Sender longArray(@NotNull CharSequence name, long[][][] values) { + if (values == null) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_LONG_ARRAY, true); + if (col != null) { + col.addLongArray(values); + } + return this; + } + + @Override + public Sender longArray(@NotNull CharSequence name, LongArray array) { + if (array == null) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_LONG_ARRAY, true); + if (col != null) { + col.addLongArray(array); + } + return this; + } + + @Override + public QwpWebSocketSender longColumn(CharSequence columnName, long value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_LONG, true); + if (col != null) { + col.addLong(value); + } + return this; + } + + @Override + public void reset() { + checkNotClosed(); + // Reset ALL table buffers, not just the current one + ObjList keys = tableBuffers.keys(); + for (int i = 0, n = keys.size(); i < n; i++) { + QwpTableBuffer buf = tableBuffers.get(keys.getQuick(i)); + if (buf != null) { + buf.reset(); + } + } + pendingBytes = 0; + pendingRowCount = 0; + firstPendingRowTimeNanos = 0; + currentTableBuffer = null; + currentTableName = null; + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; + } + + /** + * Sets whether to use Gorilla timestamp encoding. + */ + public void setGorillaEnabled(boolean enabled) { + this.gorillaEnabled = enabled; + this.encoder.setGorillaEnabled(enabled); + } + + /** + * Adds a SHORT column value to the current row. + * + * @param columnName the column name + * @param value the short value + * @return this sender for method chaining + */ + public QwpWebSocketSender shortColumn(CharSequence columnName, short value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_SHORT, false); + if (col != null) { + col.addShort(value); + } + return this; + } + + @Override + public QwpWebSocketSender stringColumn(CharSequence columnName, CharSequence value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_STRING, true); + if (col != null) { + col.addString(value); + } + return this; + } + + @Override + public QwpWebSocketSender symbol(CharSequence columnName, CharSequence value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_SYMBOL, true); + if (col != null) { + col.addSymbol(value); + } + return this; + } + + @Override + public QwpWebSocketSender table(CharSequence tableName) { + checkNotClosed(); + // Fast path: if table name matches current, skip hashmap lookup + if (currentTableName != null && currentTableBuffer != null && Chars.equals(tableName, currentTableName)) { + return this; + } + // Table changed - invalidate cached column references + validateTableName(tableName); + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; + currentTableName = tableName.toString(); + currentTableBuffer = tableBuffers.get(currentTableName); + if (currentTableBuffer == null) { + currentTableBuffer = new QwpTableBuffer(currentTableName, this); + tableBuffers.put(currentTableName, currentTableBuffer); + } + // Both modes accumulate rows until flush + return this; + } + + @Override + public QwpWebSocketSender timestampColumn(CharSequence columnName, long value, ChronoUnit unit) { + checkNotClosed(); + checkTableSelected(); + if (unit == ChronoUnit.NANOS) { + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_TIMESTAMP_NANOS, true); + if (col != null) { + col.addLong(value); + } + } else { + long micros = toMicros(value, unit); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_TIMESTAMP, true); + if (col != null) { + col.addLong(micros); + } + } + return this; + } + + @Override + public QwpWebSocketSender timestampColumn(CharSequence columnName, Instant value) { + checkNotClosed(); + checkTableSelected(); + long micros = value.getEpochSecond() * 1_000_000L + value.getNano() / 1000L; + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_TIMESTAMP, true); + if (col != null) { + col.addLong(micros); + } + return this; + } + + /** + * Adds a UUID column value to the current row. + * + * @param columnName the column name + * @param lo the low 64 bits of the UUID + * @param hi the high 64 bits of the UUID + * @return this sender for method chaining + */ + public QwpWebSocketSender uuidColumn(CharSequence columnName, long lo, long hi) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_UUID, true); + if (col != null) { + col.addUuid(hi, lo); + } + return this; + } + + private void atMicros(long timestampMicros) { + // Add designated timestamp column (empty name for designated timestamp) + // Use cached reference to avoid hashmap lookup per row + if (cachedTimestampColumn == null) { + cachedTimestampColumn = currentTableBuffer.getOrCreateDesignatedTimestampColumn(TYPE_TIMESTAMP); + } + cachedTimestampColumn.addLong(timestampMicros); + sendRow(); + } + + private void atNanos(long timestampNanos) { + // Add designated timestamp column (empty name for designated timestamp) + // Use cached reference to avoid hashmap lookup per row + if (cachedTimestampNanosColumn == null) { + cachedTimestampNanosColumn = currentTableBuffer.getOrCreateDesignatedTimestampColumn(TYPE_TIMESTAMP_NANOS); + } + cachedTimestampNanosColumn.addLong(timestampNanos); + sendRow(); + } + + private void checkNotClosed() { + if (closed) { + throw new LineSenderException("Sender is closed"); + } + } + + private void checkTableSelected() { + if (currentTableBuffer == null) { + throw new LineSenderException("table() must be called before adding columns"); + } + } + + /** + * Ensures the active buffer is ready for writing (in FILLING state). + * If the buffer is in RECYCLED state, resets it. If it's in use, waits for it. + */ + private void ensureActiveBufferReady() { + if (activeBuffer.isFilling()) { + return; // Already ready + } + + if (activeBuffer.isRecycled()) { + // Buffer was recycled but not reset - reset it now + activeBuffer.reset(); + return; + } + + // Buffer is in use (SEALED or SENDING) - wait for it + // Use a while loop to handle spurious wakeups and race conditions with the latch + while (activeBuffer.isInUse()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting for active buffer [id={}, state={}]", activeBuffer.getBatchId(), MicrobatchBuffer.stateName(activeBuffer.getState())); + } + boolean recycled = activeBuffer.awaitRecycled(30, TimeUnit.SECONDS); + if (!recycled) { + throw new LineSenderException("Timeout waiting for active buffer to be recycled"); + } + } + + // Buffer should now be RECYCLED - reset it + if (activeBuffer.isRecycled()) { + activeBuffer.reset(); + } + } + + private void ensureConnected() { + if (closed) { + throw new LineSenderException("Sender is closed"); + } + if (!connected) { + // Create WebSocket client using factory (zero-GC native implementation) + if (tlsEnabled) { + client = WebSocketClientFactory.newInsecureTlsInstance(); + } else { + client = WebSocketClientFactory.newPlainTextInstance(); + } + + // Connect and upgrade to WebSocket + try { + client.connect(host, port); + client.upgrade(WRITE_PATH, authorizationHeader); + } catch (Exception e) { + client.close(); + client = null; + throw new LineSenderException("Failed to connect to " + host + ":" + port, e); + } + + // a window for tracking batches awaiting ACK (both modes) + inFlightWindow = new InFlightWindow(inFlightWindowSize, InFlightWindow.DEFAULT_TIMEOUT_MS); + + // Initialize send queue for async mode (window > 1) + // The send queue handles both sending AND receiving (single I/O thread) + if (inFlightWindowSize > 1) { + try { + sendQueue = new WebSocketSendQueue(client, inFlightWindow, + WebSocketSendQueue.DEFAULT_ENQUEUE_TIMEOUT_MS, + WebSocketSendQueue.DEFAULT_SHUTDOWN_TIMEOUT_MS); + } catch (Throwable t) { + inFlightWindow = null; + client.close(); + client = null; + throw new LineSenderException("Failed to start I/O thread for " + host + ":" + port, t); + } + } + // Sync mode (window=1): no send queue - we send and read ACKs synchronously + + // Clear sent schema hashes - server starts fresh on each connection + sentSchemaHashes.clear(); + + connected = true; + LOG.info("Connected to WebSocket [host={}, port={}, windowSize={}]", host, port, inFlightWindowSize); + } + } + + private void failExpectedIfNeeded(long expectedSequence, LineSenderException error) { + if (inFlightWindow != null && inFlightWindow.getLastError() == null) { + inFlightWindow.fail(expectedSequence, error); + } + } + + /** + * Flushes pending rows by encoding and sending them. + * Each table's rows are encoded into a separate QWP v1 message and sent as one WebSocket frame. + */ + private void flushPendingRows() { + if (pendingRowCount <= 0) { + return; + } + + // Invalidate cached column references — table buffers will be reset below + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; + + if (LOG.isDebugEnabled()) { + LOG.debug("Flushing pending rows [count={}, tables={}]", pendingRowCount, tableBuffers.size()); + } + + // Ensure activeBuffer is ready for writing + // It might be in RECYCLED state if previous batch was sent but we didn't swap yet + ensureActiveBufferReady(); + + // Encode all table buffers that have data + // Iterate over the keys list directly + ObjList keys = tableBuffers.keys(); + for (int i = 0, n = keys.size(); i < n; i++) { + CharSequence tableName = keys.getQuick(i); + if (tableName == null) { + continue; // Skip null entries (shouldn't happen but be safe) + } + QwpTableBuffer tableBuffer = tableBuffers.get(tableName); + if (tableBuffer == null) { + continue; + } + int rowCount = tableBuffer.getRowCount(); + if (rowCount > 0) { + // Check if this schema has been sent before (use schema reference mode if so) + // Combined key includes table name since server caches by (tableName, schemaHash) + long schemaHash = tableBuffer.getSchemaHash(); + long schemaKey = schemaHash ^ ((long) tableBuffer.getTableName().hashCode() << 32); + boolean useSchemaRef = sentSchemaHashes.contains(schemaKey); + + if (LOG.isDebugEnabled()) { + LOG.debug("Encoding table [name={}, rows={}, maxSentSymbolId={}, batchMaxId={}, useSchemaRef={}]", tableName, rowCount, maxSentSymbolId, currentBatchMaxSymbolId, useSchemaRef); + } + + // Encode this table's rows with delta symbol dictionary + int messageSize = encoder.encodeWithDeltaDict( + tableBuffer, + globalSymbolDictionary, + maxSentSymbolId, + currentBatchMaxSymbolId, + useSchemaRef + ); + + QwpBufferWriter buffer = encoder.getBuffer(); + + // Copy to microbatch buffer and seal immediately + // Each QWP v1 message must be in its own WebSocket frame + activeBuffer.ensureCapacity(messageSize); + activeBuffer.write(buffer.getBufferPtr(), messageSize); + activeBuffer.incrementRowCount(); + + // Seal and enqueue for sending + sealAndSwapBuffer(); + + // Update sent state only after successful enqueue. + // If sealAndSwapBuffer() threw, these remain unchanged so the + // next batch's delta dictionary will correctly re-include the + // symbols and schema that the server never received. + maxSentSymbolId = currentBatchMaxSymbolId; + if (!useSchemaRef) { + sentSchemaHashes.add(schemaKey); + } + + // Reset table buffer and batch-level symbol tracking + tableBuffer.reset(); + currentBatchMaxSymbolId = -1; + } + } + + // Reset pending count + pendingBytes = 0; + pendingRowCount = 0; + firstPendingRowTimeNanos = 0; + } + + /** + * Flushes pending rows synchronously, blocking until server ACKs. + * Used in sync mode for simpler, blocking operation. + */ + private void flushSync() { + if (pendingRowCount <= 0) { + return; + } + + // Invalidate cached column references — table buffers will be reset below + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; + + if (LOG.isDebugEnabled()) { + LOG.debug("Sync flush [pendingRows={}, tables={}]", pendingRowCount, tableBuffers.size()); + } + + // Encode all table buffers that have data into a single message + ObjList keys = tableBuffers.keys(); + for (int i = 0, n = keys.size(); i < n; i++) { + CharSequence tableName = keys.getQuick(i); + if (tableName == null) { + continue; + } + QwpTableBuffer tableBuffer = tableBuffers.get(tableName); + if (tableBuffer == null || tableBuffer.getRowCount() == 0) { + continue; + } + + // Check if this schema has been sent before (use schema reference mode if so) + // Combined key includes table name since server caches by (tableName, schemaHash) + long schemaHash = tableBuffer.getSchemaHash(); + long schemaKey = schemaHash ^ ((long) tableBuffer.getTableName().hashCode() << 32); + boolean useSchemaRef = sentSchemaHashes.contains(schemaKey); + + // Encode this table's rows with delta symbol dictionary + int messageSize = encoder.encodeWithDeltaDict( + tableBuffer, + globalSymbolDictionary, + maxSentSymbolId, + currentBatchMaxSymbolId, + useSchemaRef + ); + + if (messageSize > 0) { + QwpBufferWriter buffer = encoder.getBuffer(); + + // Track batch in InFlightWindow before sending + long batchSequence = nextBatchSequence++; + inFlightWindow.addInFlight(batchSequence); + + if (LOG.isDebugEnabled()) { + LOG.debug("Sending sync batch [seq={}, bytes={}, rows={}, maxSentSymbolId={}, useSchemaRef={}]", batchSequence, messageSize, tableBuffer.getRowCount(), currentBatchMaxSymbolId, useSchemaRef); + } + + // Send over WebSocket and fail the in-flight entry if send throws, + // so close() does not hang waiting for an ACK that will never arrive. + try { + client.sendBinary(buffer.getBufferPtr(), messageSize); + } catch (LineSenderException e) { + failExpectedIfNeeded(batchSequence, e); + throw e; + } catch (Throwable t) { + LineSenderException error = new LineSenderException("Failed to send batch " + batchSequence, t); + failExpectedIfNeeded(batchSequence, error); + throw error; + } + + // Wait for ACK synchronously + waitForAck(batchSequence); + + // Update sent state only after successful send + ACK. + // If sendBinary() or waitForAck() threw, these remain unchanged + // so the next batch's delta dictionary will correctly re-include + // the symbols and schema that the server never received. + maxSentSymbolId = currentBatchMaxSymbolId; + if (!useSchemaRef) { + sentSchemaHashes.add(schemaKey); + } + } + + // Reset table buffer after sending + tableBuffer.reset(); + + // Reset batch-level symbol tracking + currentBatchMaxSymbolId = -1; + } + + // Reset pending row tracking + pendingBytes = 0; + pendingRowCount = 0; + firstPendingRowTimeNanos = 0; + + if (LOG.isDebugEnabled()) { + LOG.debug("Sync flush complete [totalAcked={}]", inFlightWindow.getTotalAcked()); + } + } + + private long getPendingBytes() { + return pendingBytes; + } + + /** + * Seals the current buffer and swaps to the other buffer. + * Enqueues the sealed buffer for async sending. + */ + private void sealAndSwapBuffer() { + if (!activeBuffer.hasData()) { + return; // Nothing to send + } + + MicrobatchBuffer toSend = activeBuffer; + toSend.seal(); + + if (LOG.isDebugEnabled()) { + LOG.debug("Sealing buffer [id={}, rows={}, bytes={}]", toSend.getBatchId(), toSend.getRowCount(), toSend.getBufferPos()); + } + + // Swap to the other buffer + activeBuffer = (activeBuffer == buffer0) ? buffer1 : buffer0; + + // If the other buffer is still being sent, wait for it + // Use a while loop to handle spurious wakeups and race conditions with the latch + while (activeBuffer.isInUse()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting for buffer recycle [id={}, state={}]", activeBuffer.getBatchId(), MicrobatchBuffer.stateName(activeBuffer.getState())); + } + boolean recycled = activeBuffer.awaitRecycled(30, TimeUnit.SECONDS); + if (!recycled) { + throw new LineSenderException("Timeout waiting for buffer to be recycled"); + } + if (LOG.isDebugEnabled()) { + LOG.debug("Buffer recycled [id={}, state={}]", activeBuffer.getBatchId(), MicrobatchBuffer.stateName(activeBuffer.getState())); + } + } + + // Reset the new active buffer + int stateBeforeReset = activeBuffer.getState(); + if (LOG.isDebugEnabled()) { + LOG.debug("Resetting buffer [id={}, state={}]", activeBuffer.getBatchId(), MicrobatchBuffer.stateName(stateBeforeReset)); + } + activeBuffer.reset(); + + // Enqueue the sealed buffer for sending. + // If enqueue fails, roll back local state so the same batch can be retried. + try { + if (!sendQueue.enqueue(toSend)) { + throw new LineSenderException("Failed to enqueue buffer for sending"); + } + } catch (LineSenderException e) { + activeBuffer = toSend; + if (toSend.isSealed()) { + toSend.rollbackSealForRetry(); + } + throw e; + } + } + + /** + * Accumulates the current row. + * Both sync and async modes buffer rows until flush (explicit or auto-flush). + * The difference is that sync mode flush() blocks until server ACKs. + */ + private void sendRow() { + ensureConnected(); + if (autoFlushBytes > 0) { + long bytesBefore = currentTableBuffer.getBufferedBytes(); + currentTableBuffer.nextRow(); + pendingBytes += currentTableBuffer.getBufferedBytes() - bytesBefore; + } else { + currentTableBuffer.nextRow(); + } + + // Both modes: accumulate rows, don't encode yet + if (pendingRowCount == 0) { + firstPendingRowTimeNanos = System.nanoTime(); + } + pendingRowCount++; + + // Check if any flush threshold is exceeded + if (shouldAutoFlush()) { + if (inFlightWindowSize > 1) { + flushPendingRows(); + } else { + // Sync mode (window=1): flush directly with ACK wait + flushSync(); + } + } + } + + /** + * Checks if any auto-flush threshold is exceeded. + */ + private boolean shouldAutoFlush() { + if (pendingRowCount <= 0) { + return false; + } + if (autoFlushRows > 0 && pendingRowCount >= autoFlushRows) { + return true; + } + if (autoFlushBytes > 0 && getPendingBytes() >= autoFlushBytes) { + return true; + } + if (autoFlushIntervalNanos > 0) { + long ageNanos = System.nanoTime() - firstPendingRowTimeNanos; + return ageNanos >= autoFlushIntervalNanos; + } + return false; + } + + private long toMicros(long value, ChronoUnit unit) { + switch (unit) { + case NANOS: + return value / 1000L; + case MICROS: + return value; + case MILLIS: + return value * 1000L; + case SECONDS: + return value * 1_000_000L; + case MINUTES: + return value * 60_000_000L; + case HOURS: + return value * 3_600_000_000L; + case DAYS: + return value * 86_400_000_000L; + default: + throw new LineSenderException("Unsupported time unit: " + unit); + } + } + + private void validateTableName(CharSequence name) { + if (name == null || !TableUtils.isValidTableName(name, MAX_TABLE_NAME_LENGTH)) { + if (name == null || name.length() == 0) { + throw new LineSenderException("table name cannot be empty"); + } + if (name.length() > MAX_TABLE_NAME_LENGTH) { + throw new LineSenderException("table name too long [maxLength=" + MAX_TABLE_NAME_LENGTH + "]"); + } + throw new LineSenderException("table name contains illegal characters: " + name); + } + } + + /** + * Waits synchronously for an ACK from the server for the specified batch. + */ + private void waitForAck(long expectedSequence) { + long deadline = System.currentTimeMillis() + InFlightWindow.DEFAULT_TIMEOUT_MS; + + while (System.currentTimeMillis() < deadline) { + try { + sawBinaryAck = false; + boolean received = client.receiveFrame(ackHandler, 1000); // 1 second timeout per read attempt + + if (received) { + // Non-binary frames (e.g. ping/pong/text) are not ACKs. + if (!sawBinaryAck) { + continue; + } + long sequence = ackResponse.getSequence(); + if (ackResponse.isSuccess()) { + // Cumulative ACK - acknowledge all batches up to this sequence + inFlightWindow.acknowledgeUpTo(sequence); + if (sequence >= expectedSequence) { + return; // Our batch was acknowledged (cumulative) + } + // Got ACK for lower sequence - continue waiting + } else { + String errorMessage = ackResponse.getErrorMessage(); + LineSenderException error = new LineSenderException( + "Server error for batch " + sequence + ": " + + ackResponse.getStatusName() + " - " + errorMessage); + inFlightWindow.fail(sequence, error); + if (sequence == expectedSequence) { + throw error; + } + } + } + } catch (LineSenderException e) { + failExpectedIfNeeded(expectedSequence, e); + throw e; + } catch (Exception e) { + LineSenderException wrapped = new LineSenderException("Error waiting for ACK: " + e.getMessage(), e); + failExpectedIfNeeded(expectedSequence, wrapped); + throw wrapped; + } + } + + LineSenderException timeout = new LineSenderException("Timeout waiting for ACK for batch " + expectedSequence); + failExpectedIfNeeded(expectedSequence, timeout); + throw timeout; + } + + private static class AckFrameHandler implements WebSocketFrameHandler { + private final QwpWebSocketSender sender; + + AckFrameHandler(QwpWebSocketSender sender) { + this.sender = sender; + } + + @Override + public void onBinaryMessage(long payloadPtr, int payloadLen) { + sender.sawBinaryAck = true; + if (!WebSocketResponse.isStructurallyValid(payloadPtr, payloadLen)) { + throw new LineSenderException( + "Invalid ACK response payload [length=" + payloadLen + ']' + ); + } + if (!sender.ackResponse.readFrom(payloadPtr, payloadLen)) { + throw new LineSenderException("Failed to parse ACK response"); + } + } + + @Override + public void onClose(int code, String reason) { + throw new LineSenderException("WebSocket closed while waiting for ACK: " + reason); + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/SegmentedNativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/SegmentedNativeBufferWriter.java new file mode 100644 index 0000000..f7ca788 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/SegmentedNativeBufferWriter.java @@ -0,0 +1,183 @@ +/* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.std.ObjList; +import io.questdb.client.std.QuietCloseable; + +final class SegmentedNativeBufferWriter implements QwpBufferWriter, QuietCloseable { + private final ObjList chunks = new ObjList<>(); + private final NativeSegmentList segments = new NativeSegmentList(); + + private NativeBufferWriter currentChunk; + private long flushedBytes; + private int nextChunkIndex; + + SegmentedNativeBufferWriter() { + currentChunk = new NativeBufferWriter(); + chunks.add(currentChunk); + } + + @Override + public void close() { + for (int i = 0, n = chunks.size(); i < n; i++) { + chunks.getQuick(i).close(); + } + chunks.clear(); + segments.close(); + } + + void finish() { + flushCurrentChunk(); + } + + NativeSegmentList getSegments() { + return segments; + } + + @Override + public void ensureCapacity(int additionalBytes) { + currentChunk.ensureCapacity(additionalBytes); + } + + @Override + public long getBufferPtr() { + return currentChunk.getBufferPtr(); + } + + @Override + public int getCapacity() { + return currentChunk.getCapacity(); + } + + @Override + public int getPosition() { + return (int) (flushedBytes + currentChunk.getPosition()); + } + + @Override + public void patchInt(int offset, int value) { + if (offset < flushedBytes || offset + Integer.BYTES > flushedBytes + currentChunk.getPosition()) { + throw new UnsupportedOperationException("cannot patch flushed segment data"); + } + currentChunk.patchInt((int) (offset - flushedBytes), value); + } + + @Override + public void putBlockOfBytes(long from, long len) { + flushCurrentChunk(); + segments.add(from, len); + flushedBytes += len; + } + + @Override + public void putByte(byte value) { + currentChunk.putByte(value); + } + + @Override + public void putDouble(double value) { + currentChunk.putDouble(value); + } + + @Override + public void putFloat(float value) { + currentChunk.putFloat(value); + } + + @Override + public void putInt(int value) { + currentChunk.putInt(value); + } + + @Override + public void putLong(long value) { + currentChunk.putLong(value); + } + + @Override + public void putLongBE(long value) { + currentChunk.putLongBE(value); + } + + @Override + public void putShort(short value) { + currentChunk.putShort(value); + } + + @Override + public void putString(String value) { + currentChunk.putString(value); + } + + @Override + public void putUtf8(String value) { + currentChunk.putUtf8(value); + } + + @Override + public void putVarint(long value) { + currentChunk.putVarint(value); + } + + @Override + public void reset() { + segments.reset(); + flushedBytes = 0; + nextChunkIndex = 0; + for (int i = 0, n = chunks.size(); i < n; i++) { + chunks.getQuick(i).reset(); + } + currentChunk = chunks.getQuick(0); + } + + @Override + public void skip(int bytes) { + currentChunk.skip(bytes); + } + + private void flushCurrentChunk() { + int chunkSize = currentChunk.getPosition(); + if (chunkSize == 0) { + return; + } + + segments.add(currentChunk.getBufferPtr(), chunkSize); + flushedBytes += chunkSize; + currentChunk = nextChunk(); + } + + private NativeBufferWriter nextChunk() { + nextChunkIndex++; + if (nextChunkIndex < chunks.size()) { + NativeBufferWriter chunk = chunks.getQuick(nextChunkIndex); + chunk.reset(); + return chunk; + } + + NativeBufferWriter chunk = new NativeBufferWriter(); + chunks.add(chunk); + return chunk; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java new file mode 100644 index 0000000..b583b9c --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java @@ -0,0 +1,274 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.std.Unsafe; + +import java.nio.charset.StandardCharsets; + +/** + * Binary response format for WebSocket QWP v1 protocol. + *

    + * Response format (little-endian): + * 

    + * +--------+----------+------------------+
+ * | status | sequence | error (if any)   |
+ * | 1 byte | 8 bytes  | 2 bytes + UTF-8  |
+ * +--------+----------+------------------+
+ *
    + *

    + * Status codes: + *

      + *
    • 0: Success (ACK)
      • + *
    • 1: Parse error
      • + *
    • 2: Schema error
      • + *
    • 3: Write error
      • + *
    • 4: Security error
      • + *
    • 255: Internal error
      • + *
    + *

    + * The sequence number allows correlation with the original request. + * Error message is only present when status != 0. + */ +public class WebSocketResponse { + + public static final int MAX_ERROR_MESSAGE_LENGTH = 1024; + public static final int MIN_ERROR_RESPONSE_SIZE = 11; // status + sequence + error length + // Minimum response size: status (1) + sequence (8) + public static final int MIN_RESPONSE_SIZE = 9; + public static final byte STATUS_INTERNAL_ERROR = (byte) 255; + // Status codes + public static final byte STATUS_OK = 0; + public static final byte STATUS_PARSE_ERROR = 1; + public static final byte STATUS_SCHEMA_ERROR = 2; + public static final byte STATUS_SECURITY_ERROR = 4; + public static final byte STATUS_WRITE_ERROR = 3; + private String errorMessage; + private long sequence; + private byte status; + + public WebSocketResponse() { + this.status = STATUS_OK; + this.sequence = 0; + this.errorMessage = null; + } + + /** + * Creates an error response. + */ + public static WebSocketResponse error(long sequence, byte status, String errorMessage) { + WebSocketResponse response = new WebSocketResponse(); + response.status = status; + response.sequence = sequence; + response.errorMessage = errorMessage; + return response; + } + + /** + * Validates binary response framing without allocating. + *

    + * Accepted formats: + *

      + *
    • OK: exactly 9 bytes (status + sequence)
      • + *
    • Error: exactly 11 + errorLength bytes
      • + *
    + * + * @param ptr response buffer pointer + * @param length response frame payload length + * @return true if payload structure is valid + */ + public static boolean isStructurallyValid(long ptr, int length) { + if (length < MIN_RESPONSE_SIZE) { + return false; + } + + byte status = Unsafe.getUnsafe().getByte(ptr); + if (status == STATUS_OK) { + return length == MIN_RESPONSE_SIZE; + } + + if (length < MIN_ERROR_RESPONSE_SIZE) { + return false; + } + + int msgLen = Unsafe.getUnsafe().getShort(ptr + MIN_RESPONSE_SIZE) & 0xFFFF; + return length == MIN_ERROR_RESPONSE_SIZE + msgLen; + } + + /** + * Creates a success response. + */ + public static WebSocketResponse success(long sequence) { + WebSocketResponse response = new WebSocketResponse(); + response.status = STATUS_OK; + response.sequence = sequence; + return response; + } + + /** + * Returns the error message, or null for success responses. + */ + public String getErrorMessage() { + return errorMessage; + } + + /** + * Returns the sequence number. + */ + public long getSequence() { + return sequence; + } + + /** + * Returns a human-readable status name. + */ + public String getStatusName() { + switch (status) { + case STATUS_OK: + return "OK"; + case STATUS_PARSE_ERROR: + return "PARSE_ERROR"; + case STATUS_SCHEMA_ERROR: + return "SCHEMA_ERROR"; + case STATUS_WRITE_ERROR: + return "WRITE_ERROR"; + case STATUS_SECURITY_ERROR: + return "SECURITY_ERROR"; + case STATUS_INTERNAL_ERROR: + return "INTERNAL_ERROR"; + default: + return "UNKNOWN(" + (status & 0xFF) + ")"; + } + } + + /** + * Returns true if this is a success response. + */ + public boolean isSuccess() { + return status == STATUS_OK; + } + + /** + * Reads a response from native memory. + * + * @param ptr source address + * @param length available bytes + * @return true if successfully parsed, false if not enough data + */ + public boolean readFrom(long ptr, int length) { + if (length < MIN_RESPONSE_SIZE) { + return false; + } + + int offset = 0; + + // Status (1 byte) + status = Unsafe.getUnsafe().getByte(ptr + offset); + offset += 1; + + // Sequence (8 bytes, little-endian) + sequence = Unsafe.getUnsafe().getLong(ptr + offset); + offset += 8; + + // Error message (if status != OK and more data available) + if (status != STATUS_OK && length > offset + 2) { + int msgLen = Unsafe.getUnsafe().getShort(ptr + offset) & 0xFFFF; + offset += 2; + + if (length >= offset + msgLen && msgLen > 0) { + byte[] msgBytes = new byte[msgLen]; + for (int i = 0; i < msgLen; i++) { + msgBytes[i] = Unsafe.getUnsafe().getByte(ptr + offset + i); + } + errorMessage = new String(msgBytes, StandardCharsets.UTF_8); + } else { + errorMessage = null; + } + } else { + errorMessage = null; + } + + return true; + } + + /** + * Calculates the serialized size of this response. + */ + public int serializedSize() { + int size = MIN_RESPONSE_SIZE; + if (errorMessage != null && !errorMessage.isEmpty()) { + byte[] msgBytes = errorMessage.getBytes(StandardCharsets.UTF_8); + int msgLen = Math.min(msgBytes.length, MAX_ERROR_MESSAGE_LENGTH); + size += 2 + msgLen; // 2 bytes for length prefix + } + return size; + } + + @Override + public String toString() { + if (isSuccess()) { + return "WebSocketResponse{status=OK, seq=" + sequence + "}"; + } else { + return "WebSocketResponse{status=" + getStatusName() + ", seq=" + sequence + + ", error=" + errorMessage + "}"; + } + } + + /** + * Writes this response to native memory. + * + * @param ptr destination address + * @return number of bytes written + */ + public int writeTo(long ptr) { + int offset = 0; + + // Status (1 byte) + Unsafe.getUnsafe().putByte(ptr + offset, status); + offset += 1; + + // Sequence (8 bytes, little-endian) + Unsafe.getUnsafe().putLong(ptr + offset, sequence); + offset += 8; + + // Error message (if any) + if (status != STATUS_OK && errorMessage != null && !errorMessage.isEmpty()) { + byte[] msgBytes = errorMessage.getBytes(StandardCharsets.UTF_8); + int msgLen = Math.min(msgBytes.length, MAX_ERROR_MESSAGE_LENGTH); + + // Length prefix (2 bytes, little-endian) + Unsafe.getUnsafe().putShort(ptr + offset, (short) msgLen); + offset += 2; + + // Message bytes + for (int i = 0; i < msgLen; i++) { + Unsafe.getUnsafe().putByte(ptr + offset + i, msgBytes[i]); + } + offset += msgLen; + } + + return offset; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java new file mode 100644 index 0000000..9ffda54 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java @@ -0,0 +1,684 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.std.QuietCloseable; +import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Asynchronous I/O handler for WebSocket microbatch transmission. + *

    + * This class manages a dedicated I/O thread that handles both: + *

      + *
    • Sending batches via a single-slot handoff (volatile reference)
      • + *
    • Receiving and processing server ACK responses
      • + *
    + * The single-slot design matches the double-buffering scheme: at most one + * sealed buffer is pending while the other is being filled. + * Using a single thread eliminates concurrency issues with the WebSocket channel. + *

    + * Thread safety: + *

      + *
    • The pending slot is thread-safe for concurrent access
      • + *
    • Only the I/O thread interacts with the WebSocket channel
      • + *
    • Buffer state transitions ensure safe hand-over
      • + *
    + *

    + * Backpressure: + *

      + *
    • When the slot is occupied, {@link #enqueue} blocks
      • + *
    • This propagates backpressure to the user thread
      • + *
    + */ +public class WebSocketSendQueue implements QuietCloseable { + + private static final int DRAIN_SPIN_TRIES = 16; + public static final long DEFAULT_ENQUEUE_TIMEOUT_MS = 30_000; + public static final long DEFAULT_SHUTDOWN_TIMEOUT_MS = 10_000; + private static final Logger LOG = LoggerFactory.getLogger(WebSocketSendQueue.class); + // The WebSocket client for I/O (single-threaded access only) + private final WebSocketClient client; + // Configuration + private final long enqueueTimeoutMs; + // Optional InFlightWindow for tracking sent batches awaiting ACK + @Nullable + private final InFlightWindow inFlightWindow; + + // The I/O thread for async send/receive + private final Thread ioThread; + // Counter for batches currently being processed by the I/O thread + // This tracks batches that have been dequeued but not yet fully sent + private final AtomicInteger processingCount = new AtomicInteger(0); + // Lock for all coordination between user thread and I/O thread. + // Used for: queue poll + processingCount increment atomicity, + // flush() waiting, I/O thread waiting when idle. + private final Object processingLock = new Object(); + // Response parsing + private final WebSocketResponse response = new WebSocketResponse(); + private final ResponseHandler responseHandler = new ResponseHandler(); + // Synchronization for flush/close + private final CountDownLatch shutdownLatch; + private final long shutdownTimeoutMs; + // Statistics - receiving + private final AtomicLong totalAcks = new AtomicLong(0); + // Statistics - sending + private final AtomicLong totalBatchesSent = new AtomicLong(0); + private final AtomicLong totalBytesSent = new AtomicLong(0); + private final AtomicLong totalErrors = new AtomicLong(0); + // Error handling + private volatile Throwable lastError; + // Batch sequence counter (must match server's messageSequence) + private long nextBatchSequence = 0; + // Single pending buffer slot (double-buffering means at most 1 item in queue) + // Zero allocation - just a volatile reference handoff + private volatile MicrobatchBuffer pendingBuffer; + // Running state + private volatile boolean running; + private volatile boolean shuttingDown; + + /** + * Creates a new send queue with default configuration. + * + * @param client the WebSocket client for I/O + */ + public WebSocketSendQueue(WebSocketClient client) { + this(client, null, DEFAULT_ENQUEUE_TIMEOUT_MS, DEFAULT_SHUTDOWN_TIMEOUT_MS); + } + + /** + * Creates a new send queue with InFlightWindow for tracking sent batches. + * + * @param client the WebSocket client for I/O + * @param inFlightWindow the window to track sent batches awaiting ACK (may be null) + */ + public WebSocketSendQueue(WebSocketClient client, @Nullable InFlightWindow inFlightWindow) { + this(client, inFlightWindow, DEFAULT_ENQUEUE_TIMEOUT_MS, DEFAULT_SHUTDOWN_TIMEOUT_MS); + } + + /** + * Creates a new send queue with custom configuration. + * + * @param client the WebSocket client for I/O + * @param inFlightWindow the window to track sent batches awaiting ACK (may be null) + * @param enqueueTimeoutMs timeout for enqueue operations (ms) + * @param shutdownTimeoutMs timeout for graceful shutdown (ms) + */ + public WebSocketSendQueue(WebSocketClient client, @Nullable InFlightWindow inFlightWindow, + long enqueueTimeoutMs, long shutdownTimeoutMs) { + if (client == null) { + throw new IllegalArgumentException("client cannot be null"); + } + + this.client = client; + this.inFlightWindow = inFlightWindow; + this.enqueueTimeoutMs = enqueueTimeoutMs; + this.shutdownTimeoutMs = shutdownTimeoutMs; + this.running = true; + this.shuttingDown = false; + this.shutdownLatch = new CountDownLatch(1); + + // Start the I/O thread (handles both sending and receiving) + this.ioThread = new Thread(this::ioLoop, "questdb-websocket-io"); + this.ioThread.setDaemon(true); + this.ioThread.start(); + + LOG.info("WebSocket I/O thread started"); + } + + /** + * Closes the send queue gracefully. + *

    + * This method: + * 1. Stops accepting new batches + * 2. Waits for pending batches to be sent + * 3. Stops the I/O thread + *

    + * Note: This does NOT close the WebSocket channel - that's the caller's responsibility. + */ + @Override + public void close() { + if (!running) { + return; + } + + LOG.info("Closing WebSocket send queue [pending={}]", getPendingSize()); + + // Signal shutdown + shuttingDown = true; + + // Wait for pending batches to be sent + long startTime = System.currentTimeMillis(); + synchronized (processingLock) { + while (!isPendingEmpty()) { + long elapsed = System.currentTimeMillis() - startTime; + if (elapsed >= shutdownTimeoutMs) { + LOG.error("Shutdown timeout, {} batches not sent", getPendingSize()); + break; + } + try { + processingLock.wait(shutdownTimeoutMs - elapsed); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + } + + // Stop the I/O thread + running = false; + + // Wake up I/O thread if it's blocked on processingLock.wait() + synchronized (processingLock) { + processingLock.notifyAll(); + } + ioThread.interrupt(); + + // Wait for I/O thread to finish + try { + shutdownLatch.await(shutdownTimeoutMs, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + LOG.info("WebSocket send queue closed [totalBatches={}, totalBytes={}]", totalBatchesSent.get(), totalBytesSent.get()); + } + + /** + * Enqueues a sealed buffer for sending. + *

    + * The buffer must be in SEALED state. After this method returns successfully, + * ownership of the buffer transfers to the send queue. + * + * @param buffer the sealed buffer to send + * @return true if enqueued successfully + * @throws LineSenderException if the buffer is not sealed or an error occurred + */ + public boolean enqueue(MicrobatchBuffer buffer) { + if (buffer == null) { + throw new IllegalArgumentException("buffer cannot be null"); + } + if (!buffer.isSealed()) { + throw new LineSenderException("Buffer must be sealed before enqueue, state=" + + MicrobatchBuffer.stateName(buffer.getState())); + } + if (!running || shuttingDown) { + throw new LineSenderException("Send queue is not running"); + } + + // Check for errors from I/O thread + checkError(); + + final long deadline = System.currentTimeMillis() + enqueueTimeoutMs; + synchronized (processingLock) { + while (true) { + if (!running || shuttingDown) { + throw new LineSenderException("Send queue is not running"); + } + checkError(); + + if (offerPending(buffer)) { + processingLock.notifyAll(); + break; + } + + long remaining = deadline - System.currentTimeMillis(); + if (remaining <= 0) { + throw new LineSenderException("Enqueue timeout after " + enqueueTimeoutMs + "ms"); + } + try { + processingLock.wait(Math.min(10, remaining)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new LineSenderException("Interrupted while enqueueing", e); + } + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("Enqueued batch [id={}, bytes={}, rows={}]", buffer.getBatchId(), buffer.getBufferPos(), buffer.getRowCount()); + } + return true; + } + + /** + * Waits for all pending batches to be sent. + *

    + * This method blocks until the queue is empty and all in-flight sends complete. + * It does not close the queue - new batches can still be enqueued after flush. + * + * @throws LineSenderException if an error occurs during flush + */ + public void flush() { + checkError(); + + long startTime = System.currentTimeMillis(); + + // Wait under lock until the queue becomes empty and no batch is being sent. + synchronized (processingLock) { + while (running) { + // Atomically check: queue empty AND not processing + if (isPendingEmpty() && processingCount.get() == 0) { + break; // All done + } + + long remaining = enqueueTimeoutMs - (System.currentTimeMillis() - startTime); + if (remaining <= 0) { + throw new LineSenderException("Flush timeout after " + enqueueTimeoutMs + "ms, " + + "queue=" + getPendingSize() + ", processing=" + processingCount.get()); + } + + try { + processingLock.wait(remaining); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new LineSenderException("Interrupted while flushing", e); + } + + // Check for errors + checkError(); + } + } + + // If loop exited because running=false we still need to surface the root cause. + checkError(); + + LOG.debug("Flush complete"); + } + + /** + * Waits for all in-flight batches to be acknowledged. + */ + public void awaitPendingAcks() { + if (inFlightWindow == null) { + return; + } + + checkError(); + inFlightWindow.awaitEmpty(); + checkError(); + } + + /** + * Returns the last error that occurred in the I/O thread, or null if no error. + */ + public Throwable getLastError() { + return lastError; + } + + /** + * Returns the total number of batches sent. + */ + public long getTotalBatchesSent() { + return totalBatchesSent.get(); + } + + /** + * Returns the total number of bytes sent. + */ + public long getTotalBytesSent() { + return totalBytesSent.get(); + } + + /** + * Checks if an error occurred in the I/O thread and throws if so. + */ + private void checkError() { + Throwable error = lastError; + if (error != null) { + throw new LineSenderException("Error in send queue I/O thread: " + error.getMessage(), error); + } + } + + /** + * Computes the current I/O state based on queue and in-flight status. + */ + private IoState computeState(boolean hasInFlight) { + if (!isPendingEmpty()) { + return IoState.ACTIVE; + } else if (hasInFlight) { + return IoState.DRAINING; + } else { + return IoState.IDLE; + } + } + + private void failTransport(LineSenderException error) { + Throwable rootError = lastError; + if (rootError == null) { + lastError = error; + rootError = error; + } + running = false; + shuttingDown = true; + if (inFlightWindow != null) { + inFlightWindow.failAll(rootError); + } + synchronized (processingLock) { + MicrobatchBuffer dropped = pollPending(); + if (dropped != null) { + if (dropped.isSealed()) { + dropped.markSending(); + } + if (dropped.isSending()) { + dropped.markRecycled(); + } + } + processingLock.notifyAll(); + } + } + + private int getPendingSize() { + return pendingBuffer == null ? 0 : 1; + } + + private int idleDuringDrain(int idleCycles) { + if (idleCycles < DRAIN_SPIN_TRIES) { + Thread.onSpinWait(); + return idleCycles + 1; + } + Thread.yield(); + return DRAIN_SPIN_TRIES; + } + + /** + * The main I/O loop that handles both sending batches and receiving ACKs. + *

    + * Uses a state machine: + *

      + *
    • IDLE: block on processingLock.wait() until work arrives
      • + *
    • ACTIVE: non-blocking poll queue, send batches, check for ACKs
      • + *
    • DRAINING: no batches but ACKs pending - poll for ACKs with non-blocking backoff
      • + *
    + */ + private void ioLoop() { + LOG.info("I/O loop started"); + + try { + int drainIdleCycles = 0; + while (running || !isPendingEmpty()) { + MicrobatchBuffer batch = null; + boolean hasInFlight = (inFlightWindow != null && inFlightWindow.getInFlightCount() > 0); + IoState state = computeState(hasInFlight); + boolean receivedAcks = false; + + switch (state) { + case IDLE: + drainIdleCycles = 0; + // Nothing to do - wait for work under lock + synchronized (processingLock) { + // Re-check under lock to avoid missed wakeup + if (isPendingEmpty() && running) { + try { + processingLock.wait(100); + } catch (InterruptedException e) { + if (!running) return; + } + } + } + break; + + case ACTIVE: + case DRAINING: + // Try to receive any pending ACKs (non-blocking) + if (hasInFlight && client.isConnected()) { + receivedAcks = tryReceiveAcks(); + } + + // Try to dequeue and send a batch + boolean hasWindowSpace = (inFlightWindow == null || inFlightWindow.hasWindowSpace()); + if (hasWindowSpace) { + // Atomically: poll queue + increment processingCount + synchronized (processingLock) { + batch = pollPending(); + if (batch != null) { + processingCount.incrementAndGet(); + } + } + + if (batch != null) { + try { + safeSendBatch(batch); + } finally { + // Atomically: decrement + notify flush() + synchronized (processingLock) { + processingCount.decrementAndGet(); + processingLock.notifyAll(); + } + } + } + } + + // In DRAINING state with no work, stay non-blocking and use + // a simple spin/yield backoff. + if (state == IoState.DRAINING && batch == null) { + if (receivedAcks) { + drainIdleCycles = 0; + } else { + drainIdleCycles = idleDuringDrain(drainIdleCycles); + } + } else { + drainIdleCycles = 0; + } + break; + } + } + } finally { + shutdownLatch.countDown(); + LOG.info("I/O loop stopped [totalAcks={}, totalErrors={}]", totalAcks.get(), totalErrors.get()); + } + } + + private boolean isPendingEmpty() { + return pendingBuffer == null; + } + + private boolean offerPending(MicrobatchBuffer buffer) { + if (pendingBuffer != null) { + return false; // slot occupied + } + pendingBuffer = buffer; + return true; + } + + private MicrobatchBuffer pollPending() { + MicrobatchBuffer buffer = pendingBuffer; + if (buffer != null) { + pendingBuffer = null; + } + return buffer; + } + + /** + * Sends a batch with error handling. Does NOT manage processingCount. + */ + private void safeSendBatch(MicrobatchBuffer batch) { + try { + sendBatch(batch); + } catch (Throwable t) { + LOG.error("Error sending batch [id={}]{}", batch.getBatchId(), "", t); + failTransport(new LineSenderException("Error sending batch " + batch.getBatchId() + ": " + t.getMessage(), t)); + // Mark as recycled even on error to allow cleanup + if (batch.isSealed()) { + batch.markSending(); + } + if (batch.isSending()) { + batch.markRecycled(); + } + } + } + + /** + * Sends a single batch over the WebSocket channel. + */ + private void sendBatch(MicrobatchBuffer batch) { + // Transition state: SEALED -> SENDING + batch.markSending(); + + // Use our own sequence counter (must match server's messageSequence) + long batchSequence = nextBatchSequence++; + int bytes = batch.getBufferPos(); + int rows = batch.getRowCount(); + + if (LOG.isDebugEnabled()) { + LOG.debug("Sending batch [seq={}, bytes={}, rows={}, bufferId={}]", batchSequence, bytes, rows, batch.getBatchId()); + } + + // Add to in-flight window BEFORE sending (so we're ready for ACK) + // Use non-blocking tryAddInFlight since we already checked window space in ioLoop + if (inFlightWindow != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Adding to in-flight window [seq={}, inFlight={}, max={}]", batchSequence, inFlightWindow.getInFlightCount(), inFlightWindow.getMaxWindowSize()); + } + if (!inFlightWindow.tryAddInFlight(batchSequence)) { + // Should not happen since we checked hasWindowSpace before polling + throw new LineSenderException("In-flight window unexpectedly full"); + } + if (LOG.isDebugEnabled()) { + LOG.debug("Added to in-flight window [seq={}]", batchSequence); + } + } + + // Send over WebSocket + if (LOG.isDebugEnabled()) { + LOG.debug("Calling sendBinary [seq={}]", batchSequence); + } + client.sendBinary(batch.getBufferPtr(), bytes); + if (LOG.isDebugEnabled()) { + LOG.debug("sendBinary returned [seq={}]", batchSequence); + } + + // Update statistics + totalBatchesSent.incrementAndGet(); + totalBytesSent.addAndGet(bytes); + + // Transition state: SENDING -> RECYCLED + batch.markRecycled(); + + if (LOG.isDebugEnabled()) { + LOG.debug("Batch sent and recycled [seq={}, bufferId={}]", batchSequence, batch.getBatchId()); + } + } + + /** + * Tries to receive ACKs from the server (non-blocking). + */ + private boolean tryReceiveAcks() { + boolean received = false; + try { + while (client.tryReceiveFrame(responseHandler)) { + received = true; + // Drain all buffered ACKs before returning to the I/O loop. + } + } catch (Exception e) { + if (running) { + LOG.error("Error receiving response: {}", e.getMessage()); + failTransport(new LineSenderException("Error receiving response: " + e.getMessage(), e)); + } + } + return received; + } + + /** + * I/O loop states for the state machine. + *
      + *
    • IDLE: queue empty, no in-flight batches - can block waiting for work
      • + *
    • ACTIVE: have batches to send - non-blocking loop
      • + *
    • DRAINING: queue empty but ACKs pending - poll for ACKs with non-blocking backoff
      • + *
    + */ + private enum IoState { + IDLE, ACTIVE, DRAINING + } + + /** + * Handler for received WebSocket frames (ACKs from server). + */ + private class ResponseHandler implements WebSocketFrameHandler { + + @Override + public void onBinaryMessage(long payloadPtr, int payloadLen) { + if (!WebSocketResponse.isStructurallyValid(payloadPtr, payloadLen)) { + LineSenderException error = new LineSenderException( + "Invalid ACK response payload [length=" + payloadLen + ']' + ); + LOG.error("Invalid ACK response payload [length={}]", payloadLen); + failTransport(error); + return; + } + + // Parse response from binary payload + if (!response.readFrom(payloadPtr, payloadLen)) { + LineSenderException error = new LineSenderException("Failed to parse ACK response"); + LOG.error("Failed to parse response"); + failTransport(error); + return; + } + + long sequence = response.getSequence(); + + if (response.isSuccess()) { + // Cumulative ACK - acknowledge all batches up to this sequence + if (inFlightWindow != null) { + int acked = inFlightWindow.acknowledgeUpTo(sequence); + if (acked > 0) { + totalAcks.addAndGet(acked); + if (LOG.isDebugEnabled()) { + LOG.debug("Cumulative ACK received [upTo={}, acked={}]", sequence, acked); + } + } else if (LOG.isDebugEnabled()) { + LOG.debug("ACK for already-acknowledged sequences [upTo={}]", sequence); + } + } + } else { + // Error - fail the batch + String errorMessage = response.getErrorMessage(); + LOG.error("Error response [seq={}, status={}, error={}]", sequence, response.getStatusName(), errorMessage); + + if (inFlightWindow != null) { + LineSenderException error = new LineSenderException( + "Server error for batch " + sequence + ": " + + response.getStatusName() + " - " + errorMessage); + inFlightWindow.fail(sequence, error); + } + totalErrors.incrementAndGet(); + } + } + + @Override + public void onClose(int code, String reason) { + LOG.info("WebSocket closed by server [code={}, reason={}]", code, reason); + failTransport(new LineSenderException("WebSocket closed by server [code=" + code + ", reason=" + reason + ']')); + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java new file mode 100644 index 0000000..65a680b --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java @@ -0,0 +1,211 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.line.array.ArrayBufferAppender; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.Vect; + +/** + * Lightweight append-only off-heap buffer for columnar data storage. + *

    + * This buffer provides typed append operations (putByte, putShort, etc.) backed by + * native memory allocated via {@link Unsafe}. Memory is tracked under + * {@link MemoryTag#NATIVE_ILP_RSS} for precise accounting. + *

    + * Growth strategy: capacity doubles on each resize via {@link Unsafe#realloc}. + */ +public class OffHeapAppendMemory implements ArrayBufferAppender, QuietCloseable { + + private static final int DEFAULT_INITIAL_CAPACITY = 128; + private long appendAddress; + private long capacity; + private long pageAddress; + + public OffHeapAppendMemory() { + this(DEFAULT_INITIAL_CAPACITY); + } + + public OffHeapAppendMemory(long initialCapacity) { + this.capacity = Math.max(initialCapacity, 8); + this.pageAddress = Unsafe.malloc(this.capacity, MemoryTag.NATIVE_ILP_RSS); + this.appendAddress = pageAddress; + } + + /** + * Returns the address at the given byte offset from the start. + */ + public long addressOf(long offset) { + return pageAddress + offset; + } + + @Override + public void close() { + if (pageAddress != 0) { + Unsafe.free(pageAddress, capacity, MemoryTag.NATIVE_ILP_RSS); + pageAddress = 0; + appendAddress = 0; + capacity = 0; + } + } + + /** + * Returns the append offset (number of bytes written). + */ + public long getAppendOffset() { + return appendAddress - pageAddress; + } + + /** + * Sets the append position to the given byte offset. + * Used for truncateTo operations on column buffers. + */ + public void jumpTo(long offset) { + assert offset >= 0 && offset <= getAppendOffset(); + appendAddress = pageAddress + offset; + } + + /** + * Returns the base address of the buffer. + */ + public long pageAddress() { + return pageAddress; + } + + public void putBoolean(boolean value) { + putByte(value ? (byte) 1 : (byte) 0); + } + + public void putByte(byte value) { + ensureCapacity(1); + Unsafe.getUnsafe().putByte(appendAddress, value); + appendAddress++; + } + + public void putBlockOfBytes(long from, long len) { + if (len <= 0) { + return; + } + ensureCapacity(len); + Vect.memcpy(appendAddress, from, len); + appendAddress += len; + } + + public void putDouble(double value) { + ensureCapacity(8); + Unsafe.getUnsafe().putDouble(appendAddress, value); + appendAddress += 8; + } + + public void putFloat(float value) { + ensureCapacity(4); + Unsafe.getUnsafe().putFloat(appendAddress, value); + appendAddress += 4; + } + + public void putInt(int value) { + ensureCapacity(4); + Unsafe.getUnsafe().putInt(appendAddress, value); + appendAddress += 4; + } + + public void putLong(long value) { + ensureCapacity(8); + Unsafe.getUnsafe().putLong(appendAddress, value); + appendAddress += 8; + } + + public void putShort(short value) { + ensureCapacity(2); + Unsafe.getUnsafe().putShort(appendAddress, value); + appendAddress += 2; + } + + /** + * Encodes a Java String to UTF-8 directly into the off-heap buffer. + * Pre-ensures worst-case capacity to avoid per-byte checks. + */ + public void putUtf8(CharSequence value) { + if (value == null || value.length() == 0) { + return; + } + int len = value.length(); + ensureCapacity((long) len * 4); // worst case: all supplementary chars + for (int i = 0; i < len; i++) { + char c = value.charAt(i); + if (c < 0x80) { + Unsafe.getUnsafe().putByte(appendAddress++, (byte) c); + } else if (c < 0x800) { + Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0xC0 | (c >> 6))); + Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | (c & 0x3F))); + } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < len) { + char c2 = value.charAt(++i); + if (Character.isLowSurrogate(c2)) { + int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); + Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0xF0 | (codePoint >> 18))); + Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | ((codePoint >> 12) & 0x3F))); + Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | ((codePoint >> 6) & 0x3F))); + Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | (codePoint & 0x3F))); + } else { + Unsafe.getUnsafe().putByte(appendAddress++, (byte) '?'); + i--; + } + } else if (Character.isSurrogate(c)) { + Unsafe.getUnsafe().putByte(appendAddress++, (byte) '?'); + } else { + Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0xE0 | (c >> 12))); + Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | ((c >> 6) & 0x3F))); + Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | (c & 0x3F))); + } + } + } + + /** + * Advances the append position by the given number of bytes without writing. + */ + public void skip(long bytes) { + ensureCapacity(bytes); + appendAddress += bytes; + } + + /** + * Resets the append position to 0 without freeing memory. + */ + public void truncate() { + appendAddress = pageAddress; + } + + private void ensureCapacity(long needed) { + long used = appendAddress - pageAddress; + if (used + needed > capacity) { + long newCapacity = Math.max(capacity * 2, used + needed); + pageAddress = Unsafe.realloc(pageAddress, capacity, newCapacity, MemoryTag.NATIVE_ILP_RSS); + capacity = newCapacity; + appendAddress = pageAddress + used; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java new file mode 100644 index 0000000..675b3cc --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java @@ -0,0 +1,235 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.std.Unsafe; + +/** + * Bit-level writer for QWP v1 protocol. + *

    + * This class writes bits to a buffer in LSB-first order within each byte. + * Bits are packed sequentially, spanning byte boundaries as needed. + *

    + * The implementation buffers up to 64 bits before flushing to the output buffer + * to minimize memory operations. All writes are to direct memory for performance. + *

    + * Usage pattern: + * 

    + * QwpBitWriter writer = new QwpBitWriter();
+ * writer.reset(address, capacity);
+ * writer.writeBits(value, numBits);
+ * writer.writeBits(value2, numBits2);
+ * writer.flush(); // must call before reading output
+ * long bytesWritten = writer.getPosition() - address;
+ *
    + */ +public class QwpBitWriter { + + // Buffer for accumulating bits before writing + private long bitBuffer; + // Number of bits currently in the buffer (0-63) + private int bitsInBuffer; + private long currentAddress; + private long endAddress; + private long startAddress; + + /** + * Creates a new bit writer. Call {@link #reset} before use. + */ + public QwpBitWriter() { + } + + /** + * Aligns the writer to the next byte boundary by padding with zeros. + * If already byte-aligned, this is a no-op. + */ + public void alignToByte() { + if (bitsInBuffer > 0) { + flush(); + } + } + + /** + * Finishes writing and returns the number of bytes written since reset. + *

    + * This method flushes any remaining bits and returns the total byte count. + * + * @return bytes written since reset + */ + public int finish() { + flush(); + return (int) (currentAddress - startAddress); + } + + /** + * Flushes any remaining bits in the buffer to memory. + *

    + * If there are partial bits (less than 8), they are written as the last byte + * with the remaining high bits set to zero. + *

    + * Must be called before reading the output or getting the final position. + */ + public void flush() { + if (bitsInBuffer > 0) { + if (currentAddress >= endAddress) { + throw new LineSenderException("QwpBitWriter buffer overflow"); + } + Unsafe.getUnsafe().putByte(currentAddress++, (byte) bitBuffer); + bitBuffer = 0; + bitsInBuffer = 0; + } + } + + /** + * Returns the current write position (address). + * Note: Call {@link #flush()} first to ensure all buffered bits are written. + * + * @return the current address after all written data + */ + public long getPosition() { + return currentAddress; + } + + /** + * Resets the writer to write to the specified memory region. + * + * @param address the starting address + * @param capacity the maximum number of bytes to write + */ + public void reset(long address, long capacity) { + this.startAddress = address; + this.currentAddress = address; + this.endAddress = address + capacity; + this.bitBuffer = 0; + this.bitsInBuffer = 0; + } + + /** + * Writes a single bit. + * + * @param bit the bit value (0 or 1, only LSB is used) + */ + public void writeBit(int bit) { + writeBits(bit & 1, 1); + } + + /** + * Writes multiple bits from the given value. + *

    + * Bits are taken from the LSB of the value. For example, if value=0b1101 + * and numBits=4, the bits written are 1, 0, 1, 1 (LSB to MSB order). + * + * @param value the value containing the bits (LSB-aligned) + * @param numBits number of bits to write (1-64) + */ + public void writeBits(long value, int numBits) { + if (numBits <= 0 || numBits > 64) { + throw new AssertionError("Asked to write more than 64 bits of a long"); + } + + // Mask the value to only include the requested bits + if (numBits < 64) { + value &= (1L << numBits) - 1; + } + + int bitsToWrite = numBits; + + while (bitsToWrite > 0) { + // How many bits can fit into the current buffer (max 64 total) + int availableInBuffer = 64 - bitsInBuffer; + int bitsThisRound = Math.min(bitsToWrite, availableInBuffer); + + // Add bits to the buffer + long mask = bitsThisRound == 64 ? -1L : (1L << bitsThisRound) - 1; + bitBuffer |= (value & mask) << bitsInBuffer; + bitsInBuffer += bitsThisRound; + value >>>= bitsThisRound; + bitsToWrite -= bitsThisRound; + + // Flush complete bytes from the buffer + while (bitsInBuffer >= 8) { + if (currentAddress >= endAddress) { + throw new LineSenderException("QwpBitWriter buffer overflow"); + } + Unsafe.getUnsafe().putByte(currentAddress++, (byte) bitBuffer); + bitBuffer >>>= 8; + bitsInBuffer -= 8; + } + } + } + + /** + * Writes a complete byte, ensuring byte alignment first. + * + * @param value the byte value + */ + public void writeByte(int value) { + alignToByte(); + if (currentAddress >= endAddress) { + throw new LineSenderException("QwpBitWriter buffer overflow"); + } + Unsafe.getUnsafe().putByte(currentAddress++, (byte) value); + } + + /** + * Writes a complete 32-bit integer in little-endian order, ensuring byte alignment first. + * + * @param value the integer value + */ + public void writeInt(int value) { + alignToByte(); + if (currentAddress + 4 > endAddress) { + throw new LineSenderException("QwpBitWriter buffer overflow"); + } + Unsafe.getUnsafe().putInt(currentAddress, value); + currentAddress += 4; + } + + /** + * Writes a complete 64-bit long in little-endian order, ensuring byte alignment first. + * + * @param value the long value + */ + public void writeLong(long value) { + alignToByte(); + if (currentAddress + 8 > endAddress) { + throw new LineSenderException("QwpBitWriter buffer overflow"); + } + Unsafe.getUnsafe().putLong(currentAddress, value); + currentAddress += 8; + } + + /** + * Writes a signed value using two's complement representation. + * + * @param value the signed value + * @param numBits number of bits to use for the representation + */ + public void writeSigned(long value, int numBits) { + // Two's complement is automatic in Java for the bit pattern + writeBits(value, numBits); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java new file mode 100644 index 0000000..eef6bf0 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java @@ -0,0 +1,118 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.protocol; + +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_BOOLEAN; +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_CHAR; + +/** + * Represents a column definition in a QWP v1 schema. + *

    + * This class is immutable and safe for caching. + */ +public final class QwpColumnDef { + private final String name; + private final byte typeCode; + + /** + * Creates a column definition. + * + * @param name the column name (UTF-8) + * @param typeCode the QWP v1 type code (0x01-0x16) + */ + public QwpColumnDef(String name, byte typeCode) { + this.name = name; + this.typeCode = typeCode; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + QwpColumnDef that = (QwpColumnDef) o; + return typeCode == that.typeCode && + name.equals(that.name); + } + + /** + * Gets the column name. + */ + public String getName() { + return name; + } + + /** + * Gets the base type code. + * + * @return type code 0x01-0x16 + */ + public byte getTypeCode() { + return typeCode; + } + + /** + * Gets the type name for display purposes. + */ + public String getTypeName() { + return QwpConstants.getTypeName(typeCode); + } + + /** + * Gets the wire type code. + * + * @return type code as sent on wire + */ + public byte getWireTypeCode() { + return typeCode; + } + + @Override + public int hashCode() { + int result = name.hashCode(); + result = 31 * result + typeCode; + return result; + } + + @Override + public String toString() { + return name + ':' + getTypeName(); + } + + /** + * Validates that this column definition has a valid type code. + * + * @throws IllegalArgumentException if type code is invalid + */ + public void validate() { + // Valid type codes: TYPE_BOOLEAN (0x01) through TYPE_CHAR (0x16) + // This includes all basic types, arrays, decimals, and char + boolean valid = (typeCode >= TYPE_BOOLEAN && typeCode <= TYPE_CHAR); + if (!valid) { + throw new IllegalArgumentException( + "invalid column type code: 0x" + Integer.toHexString(typeCode) + ); + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java new file mode 100644 index 0000000..c8685e4 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java @@ -0,0 +1,404 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.protocol; + +/** + * Constants for the QWP v1 binary protocol. + */ +public final class QwpConstants { + + /** + * Default initial receive buffer size (64 KB). + */ + public static final int DEFAULT_INITIAL_RECV_BUFFER_SIZE = 64 * 1024; + /** + * Default maximum batch size in bytes (16 MB). + */ + public static final int DEFAULT_MAX_BATCH_SIZE = 16 * 1024 * 1024; + + /** + * Maximum in-flight batches for pipelining. + */ + public static final int DEFAULT_MAX_IN_FLIGHT_BATCHES = 4; + /** + * Default maximum rows per table in a batch. + */ + public static final int DEFAULT_MAX_ROWS_PER_TABLE = 1_000_000; + /** + * Default maximum tables per batch. + */ + public static final int DEFAULT_MAX_TABLES_PER_BATCH = 256; + /** + * Flag bit: Delta symbol dictionary encoding enabled. + * When set, symbol columns use global IDs and send only new dictionary entries. + */ + public static final byte FLAG_DELTA_SYMBOL_DICT = 0x08; + /** + * Flag bit: Gorilla timestamp encoding enabled. + */ + public static final byte FLAG_GORILLA = 0x04; + + /** + * Flag bit: LZ4 compression enabled. + */ + public static final byte FLAG_LZ4 = 0x01; + + /** + * Flag bit: Zstd compression enabled. + */ + public static final byte FLAG_ZSTD = 0x02; + /** + * Mask for compression flags (bits 0-1). + */ + public static final byte FLAG_COMPRESSION_MASK = FLAG_LZ4 | FLAG_ZSTD; + /** + * Offset of flags byte in header. + */ + public static final int HEADER_OFFSET_FLAGS = 5; + /** + * Size of the message header in bytes. + */ + public static final int HEADER_SIZE = 12; + /** + * Magic bytes for capability request: "ILP?" (ASCII). + */ + public static final int MAGIC_CAPABILITY_REQUEST = 0x3F504C49; // "ILP?" in little-endian + /** + * Magic bytes for capability response: "ILP!" (ASCII). + */ + public static final int MAGIC_CAPABILITY_RESPONSE = 0x21504C49; // "ILP!" in little-endian + /** + * Magic bytes for fallback response (old server): "ILP0" (ASCII). + */ + public static final int MAGIC_FALLBACK = 0x30504C49; // "ILP0" in little-endian + /** + * Magic bytes for QWP v1 message: "QWP1" (ASCII). + */ + public static final int MAGIC_MESSAGE = 0x31505751; // "QWP1" in little-endian + /** + * Maximum columns per table (QuestDB limit). + */ + public static final int MAX_COLUMNS_PER_TABLE = 2048; + /** + * Schema mode: Full schema included. + */ + public static final byte SCHEMA_MODE_FULL = 0x00; + /** + * Schema mode: Schema reference (hash lookup). + */ + public static final byte SCHEMA_MODE_REFERENCE = 0x01; + /** + * Status: Server error. + */ + public static final byte STATUS_INTERNAL_ERROR = 0x06; + /** + * Status: Batch accepted successfully. + */ + public static final byte STATUS_OK = 0x00; + /** + * Status: Back-pressure, retry later. + */ + public static final byte STATUS_OVERLOADED = 0x07; + /** + * Status: Malformed message. + */ + public static final byte STATUS_PARSE_ERROR = 0x05; + /** + * Status: Some rows failed (partial failure). + */ + public static final byte STATUS_PARTIAL = 0x01; + /** + * Status: Column type incompatible. + */ + public static final byte STATUS_SCHEMA_MISMATCH = 0x03; + /** + * Status: Schema hash not recognized. + */ + public static final byte STATUS_SCHEMA_REQUIRED = 0x02; + /** + * Status: Table doesn't exist (auto-create disabled). + */ + public static final byte STATUS_TABLE_NOT_FOUND = 0x04; + /** + * Column type: BOOLEAN (1 bit per value, packed). + */ + public static final byte TYPE_BOOLEAN = 0x01; + /** + * Column type: BYTE (int8). + */ + public static final byte TYPE_BYTE = 0x02; + /** + * Column type: CHAR (2-byte UTF-16 code unit). + */ + public static final byte TYPE_CHAR = 0x16; + /** + * Column type: DATE (int64 milliseconds since epoch). + */ + public static final byte TYPE_DATE = 0x0B; + + /** + * Column type: DECIMAL128 (16 bytes, 38 digits precision). + * Wire format: [scale (1B in schema)] + [big-endian unscaled value (16B)] + */ + public static final byte TYPE_DECIMAL128 = 0x14; + /** + * Column type: DECIMAL256 (32 bytes, 77 digits precision). + * Wire format: [scale (1B in schema)] + [big-endian unscaled value (32B)] + */ + public static final byte TYPE_DECIMAL256 = 0x15; + + /** + * Column type: DECIMAL64 (8 bytes, 18 digits precision). + * Wire format: [scale (1B in schema)] + [big-endian unscaled value (8B)] + */ + public static final byte TYPE_DECIMAL64 = 0x13; + /** + * Column type: DOUBLE (IEEE 754 float64). + */ + public static final byte TYPE_DOUBLE = 0x07; + /** + * Column type: DOUBLE_ARRAY (N-dimensional array of IEEE 754 float64). + * Wire format: [nDims (1B)] [dim1_len (4B)]...[dimN_len (4B)] [flattened values (LE)] + */ + public static final byte TYPE_DOUBLE_ARRAY = 0x11; + /** + * Column type: FLOAT (IEEE 754 float32). + */ + public static final byte TYPE_FLOAT = 0x06; + /** + * Column type: GEOHASH (varint bits + packed geohash). + */ + public static final byte TYPE_GEOHASH = 0x0E; + /** + * Column type: INT (int32, little-endian). + */ + public static final byte TYPE_INT = 0x04; + /** + * Column type: LONG (int64, little-endian). + */ + public static final byte TYPE_LONG = 0x05; + /** + * Column type: LONG256 (32 bytes, big-endian). + */ + public static final byte TYPE_LONG256 = 0x0D; + + /** + * Column type: LONG_ARRAY (N-dimensional array of int64). + * Wire format: [nDims (1B)] [dim1_len (4B)]...[dimN_len (4B)] [flattened values (LE)] + */ + public static final byte TYPE_LONG_ARRAY = 0x12; + /** + * Column type: SHORT (int16, little-endian). + */ + public static final byte TYPE_SHORT = 0x03; + /** + * Column type: STRING (length-prefixed UTF-8). + */ + public static final byte TYPE_STRING = 0x08; + /** + * Column type: SYMBOL (dictionary-encoded string). + */ + public static final byte TYPE_SYMBOL = 0x09; + /** + * Column type: TIMESTAMP (int64 microseconds since epoch). + * Use this for timestamps beyond nanosecond range (year > 2262). + */ + public static final byte TYPE_TIMESTAMP = 0x0A; + /** + * Column type: TIMESTAMP_NANOS (int64 nanoseconds since epoch). + * Use this for full nanosecond precision (limited to years 1677-2262). + */ + public static final byte TYPE_TIMESTAMP_NANOS = 0x10; + /** + * Column type: UUID (16 bytes, big-endian). + */ + public static final byte TYPE_UUID = 0x0C; + + /** + * Column type: VARCHAR (length-prefixed UTF-8, aux storage). + */ + public static final byte TYPE_VARCHAR = 0x0F; + /** + * Current protocol version. + */ + public static final byte VERSION_1 = 1; + + private QwpConstants() { + // utility class + } + + /** + * Returns the per-value size in bytes as encoded on the wire. BOOLEAN returns 0 + * because it is bit-packed (1 bit per value). GEOHASH returns -1 because it uses + * variable-width encoding (varint precision + ceil(precision/8) bytes per value). + *

    + * This is distinct from the in-memory buffer stride used by the client's + * {@code QwpTableBuffer.elementSizeInBuffer()}. + * + * @param typeCode the column type code (without nullable flag) + * @return size in bytes, 0 for bit-packed (BOOLEAN), or -1 for variable-width types + */ + public static int getFixedTypeSize(byte typeCode) { + int code = typeCode; + switch (code) { + case TYPE_BOOLEAN: + return 0; // Special: bit-packed + case TYPE_BYTE: + return 1; + case TYPE_SHORT: + case TYPE_CHAR: + return 2; + case TYPE_INT: + case TYPE_FLOAT: + return 4; + case TYPE_LONG: + case TYPE_DOUBLE: + case TYPE_TIMESTAMP: + case TYPE_TIMESTAMP_NANOS: + case TYPE_DATE: + case TYPE_DECIMAL64: + return 8; + case TYPE_UUID: + case TYPE_DECIMAL128: + return 16; + case TYPE_LONG256: + case TYPE_DECIMAL256: + return 32; + case TYPE_GEOHASH: + return -1; // Variable width: varint precision + packed values + default: + return -1; // Variable width + } + } + + /** + * Returns a human-readable name for the type code. + * + * @param typeCode the column type code + * @return type name + */ + public static String getTypeName(byte typeCode) { + int code = typeCode; + String name; + switch (code) { + case TYPE_BOOLEAN: + name = "BOOLEAN"; + break; + case TYPE_BYTE: + name = "BYTE"; + break; + case TYPE_SHORT: + name = "SHORT"; + break; + case TYPE_CHAR: + name = "CHAR"; + break; + case TYPE_INT: + name = "INT"; + break; + case TYPE_LONG: + name = "LONG"; + break; + case TYPE_FLOAT: + name = "FLOAT"; + break; + case TYPE_DOUBLE: + name = "DOUBLE"; + break; + case TYPE_STRING: + name = "STRING"; + break; + case TYPE_SYMBOL: + name = "SYMBOL"; + break; + case TYPE_TIMESTAMP: + name = "TIMESTAMP"; + break; + case TYPE_TIMESTAMP_NANOS: + name = "TIMESTAMP_NANOS"; + break; + case TYPE_DATE: + name = "DATE"; + break; + case TYPE_UUID: + name = "UUID"; + break; + case TYPE_LONG256: + name = "LONG256"; + break; + case TYPE_GEOHASH: + name = "GEOHASH"; + break; + case TYPE_VARCHAR: + name = "VARCHAR"; + break; + case TYPE_DOUBLE_ARRAY: + name = "DOUBLE_ARRAY"; + break; + case TYPE_LONG_ARRAY: + name = "LONG_ARRAY"; + break; + case TYPE_DECIMAL64: + name = "DECIMAL64"; + break; + case TYPE_DECIMAL128: + name = "DECIMAL128"; + break; + case TYPE_DECIMAL256: + name = "DECIMAL256"; + break; + default: + name = "UNKNOWN(" + code + ")"; + break; + } + return name; + } + + /** + * Returns true if the type code represents a fixed-width type. + * + * @param typeCode the column type code (without nullable flag) + * @return true if fixed-width + */ + public static boolean isFixedWidthType(byte typeCode) { + int code = typeCode; + return code == TYPE_BOOLEAN || + code == TYPE_BYTE || + code == TYPE_SHORT || + code == TYPE_CHAR || + code == TYPE_INT || + code == TYPE_LONG || + code == TYPE_FLOAT || + code == TYPE_DOUBLE || + code == TYPE_TIMESTAMP || + code == TYPE_TIMESTAMP_NANOS || + code == TYPE_DATE || + code == TYPE_UUID || + code == TYPE_LONG256 || + code == TYPE_DECIMAL64 || + code == TYPE_DECIMAL128 || + code == TYPE_DECIMAL256; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java new file mode 100644 index 0000000..994dfaa --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java @@ -0,0 +1,295 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.std.Unsafe; + +/** + * Gorilla delta-of-delta encoder for timestamps in QWP v1 format. + *

    + * This encoder is used by the WebSocket encoder to compress timestamp columns. + * It uses delta-of-delta compression where: + * 

    + * DoD = (t[n] - t[n-1]) - (t[n-1] - t[n-2])
+ *
+ * if DoD == 0:              write '0'              (1 bit)
+ * elif DoD in [-64, 63]:    write '10' + 7-bit     (9 bits)
+ * elif DoD in [-256, 255]:  write '110' + 9-bit    (12 bits)
+ * elif DoD in [-2048, 2047]: write '1110' + 12-bit (16 bits)
+ * else:                     write '1111' + 32-bit  (36 bits)
+ *
    + *

    + * The encoder writes first two timestamps uncompressed, then encodes + * remaining timestamps using delta-of-delta compression. + */ +public class QwpGorillaEncoder { + + private static final int BUCKET_12BIT_MAX = 2047; + private static final int BUCKET_12BIT_MIN = -2048; + private static final int BUCKET_7BIT_MAX = 63; + // Bucket boundaries (two's complement signed ranges) + private static final int BUCKET_7BIT_MIN = -64; + private static final int BUCKET_9BIT_MAX = 255; + private static final int BUCKET_9BIT_MIN = -256; + private final QwpBitWriter bitWriter = new QwpBitWriter(); + + /** + * Creates a new Gorilla encoder. + */ + public QwpGorillaEncoder() { + } + + /** + * Calculates the encoded size in bytes for Gorilla-encoded timestamps stored off-heap. + *

    + * Note: This does NOT include the encoding flag byte. Add 1 byte if + * the encoding flag is needed. + * + * @param srcAddress source address of contiguous int64 timestamps in native memory + * @param count number of timestamps + * @return encoded size in bytes (excluding encoding flag) + */ + public static int calculateEncodedSize(long srcAddress, int count) { + if (count == 0) { + return 0; + } + + int size = 8; // first timestamp + + if (count == 1) { + return size; + } + + size += 8; // second timestamp + + if (count == 2) { + return size; + } + + // Calculate bits for delta-of-delta encoding + long prevTimestamp = Unsafe.getUnsafe().getLong(srcAddress + 8); + long prevDelta = prevTimestamp - Unsafe.getUnsafe().getLong(srcAddress); + int totalBits = 0; + + for (int i = 2; i < count; i++) { + long ts = Unsafe.getUnsafe().getLong(srcAddress + (long) i * 8); + long delta = ts - prevTimestamp; + long deltaOfDelta = delta - prevDelta; + + totalBits += getBitsRequired(deltaOfDelta); + + prevDelta = delta; + prevTimestamp = ts; + } + + // Round up to bytes + size += (totalBits + 7) / 8; + + return size; + } + + /** + * Checks if Gorilla encoding can be used for timestamps stored off-heap. + *

    + * Gorilla encoding uses 32-bit signed integers for delta-of-delta values, + * so it cannot encode timestamps where the delta-of-delta exceeds the + * 32-bit signed integer range. + * + * @param srcAddress source address of contiguous int64 timestamps in native memory + * @param count number of timestamps + * @return true if Gorilla encoding can be used, false otherwise + */ + public static boolean canUseGorilla(long srcAddress, int count) { + if (count < 3) { + return true; // No DoD encoding needed for 0, 1, or 2 timestamps + } + + long prevDelta = Unsafe.getUnsafe().getLong(srcAddress + 8) - Unsafe.getUnsafe().getLong(srcAddress); + for (int i = 2; i < count; i++) { + long delta = Unsafe.getUnsafe().getLong(srcAddress + (long) i * 8) + - Unsafe.getUnsafe().getLong(srcAddress + (long) (i - 1) * 8); + long dod = delta - prevDelta; + if (dod < Integer.MIN_VALUE || dod > Integer.MAX_VALUE) { + return false; + } + prevDelta = delta; + } + return true; + } + + /** + * Returns the number of bits required to encode a delta-of-delta value. + * + * @param deltaOfDelta the delta-of-delta value + * @return bits required + */ + public static int getBitsRequired(long deltaOfDelta) { + int bucket = getBucket(deltaOfDelta); + switch (bucket) { + case 0: + return 1; + case 1: + return 9; + case 2: + return 12; + case 3: + return 16; + default: + return 36; + } + } + + /** + * Determines which bucket a delta-of-delta value falls into. + * + * @param deltaOfDelta the delta-of-delta value + * @return bucket number (0 = 1-bit, 1 = 9-bit, 2 = 12-bit, 3 = 16-bit, 4 = 36-bit) + */ + public static int getBucket(long deltaOfDelta) { + if (deltaOfDelta == 0) { + return 0; // 1-bit + } else if (deltaOfDelta >= BUCKET_7BIT_MIN && deltaOfDelta <= BUCKET_7BIT_MAX) { + return 1; // 9-bit (2 prefix + 7 value) + } else if (deltaOfDelta >= BUCKET_9BIT_MIN && deltaOfDelta <= BUCKET_9BIT_MAX) { + return 2; // 12-bit (3 prefix + 9 value) + } else if (deltaOfDelta >= BUCKET_12BIT_MIN && deltaOfDelta <= BUCKET_12BIT_MAX) { + return 3; // 16-bit (4 prefix + 12 value) + } else { + return 4; // 36-bit (4 prefix + 32 value) + } + } + + /** + * Encodes a delta-of-delta value using bucket selection. + *

    + * Prefix patterns are written LSB-first to match the decoder's read order: + *

      + *
    • '0' -> write bit 0
      • + *
    • '10' -> write bit 1, then bit 0 (0b01 as 2-bit value)
      • + *
    • '110' -> write bit 1, bit 1, bit 0 (0b011 as 3-bit value)
      • + *
    • '1110' -> write bit 1, bit 1, bit 1, bit 0 (0b0111 as 4-bit value)
      • + *
    • '1111' -> write bit 1, bit 1, bit 1, bit 1 (0b1111 as 4-bit value)
      • + *
    + * + * @param deltaOfDelta the delta-of-delta value to encode + */ + public void encodeDoD(long deltaOfDelta) { + int bucket = getBucket(deltaOfDelta); + switch (bucket) { + case 0: + bitWriter.writeBit(0); + break; + case 1: + bitWriter.writeBits(0b01, 2); + bitWriter.writeSigned(deltaOfDelta, 7); + break; + case 2: + bitWriter.writeBits(0b011, 3); + bitWriter.writeSigned(deltaOfDelta, 9); + break; + case 3: + bitWriter.writeBits(0b0111, 4); + bitWriter.writeSigned(deltaOfDelta, 12); + break; + default: + bitWriter.writeBits(0b1111, 4); + bitWriter.writeSigned(deltaOfDelta, 32); + break; + } + } + + /** + * Encodes timestamps from off-heap memory using Gorilla compression. + *

    + * Format: + * 

    +     * - First timestamp: int64 (8 bytes, little-endian)
+     * - Second timestamp: int64 (8 bytes, little-endian)
+     * - Remaining timestamps: bit-packed delta-of-delta
+     *
    + *

    + * Precondition: the caller must verify that {@link #canUseGorilla(long, int)} + * returns {@code true} before calling this method. The largest delta-of-delta + * bucket uses 32-bit signed encoding, so values outside the {@code int} range + * are silently truncated, producing corrupt output on decode. + *

    + * Note: This method does NOT write the encoding flag byte. The caller is + * responsible for writing the ENCODING_GORILLA flag before calling this method. + * + * @param destAddress destination address in native memory + * @param capacity maximum number of bytes to write + * @param srcAddress source address of contiguous int64 timestamps in native memory + * @param count number of timestamps to encode + * @return number of bytes written + */ + public int encodeTimestamps(long destAddress, long capacity, long srcAddress, int count) { + if (count == 0) { + return 0; + } + + int pos; + + // Write first timestamp uncompressed + if (capacity < 8) { + throw new LineSenderException("Gorilla encoder buffer overflow"); + } + long ts0 = Unsafe.getUnsafe().getLong(srcAddress); + Unsafe.getUnsafe().putLong(destAddress, ts0); + pos = 8; + + if (count == 1) { + return pos; + } + + // Write second timestamp uncompressed + if (capacity < pos + 8) { + throw new LineSenderException("Gorilla encoder buffer overflow"); + } + long ts1 = Unsafe.getUnsafe().getLong(srcAddress + 8); + Unsafe.getUnsafe().putLong(destAddress + pos, ts1); + pos += 8; + + if (count == 2) { + return pos; + } + + // Encode remaining with delta-of-delta + bitWriter.reset(destAddress + pos, capacity - pos); + long prevTs = ts1; + long prevDelta = ts1 - ts0; + + for (int i = 2; i < count; i++) { + long ts = Unsafe.getUnsafe().getLong(srcAddress + (long) i * 8); + long delta = ts - prevTs; + long dod = delta - prevDelta; + encodeDoD(dod); + prevDelta = delta; + prevTs = ts; + } + + return pos + bitWriter.finish(); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java new file mode 100644 index 0000000..8c7db9c --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java @@ -0,0 +1,518 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.protocol; + + +import io.questdb.client.std.Unsafe; + +/** + * XXHash64 implementation for schema hashing in QWP v1 protocol. + *

    + * The schema hash is computed over column definitions (name + type) to enable + * schema caching. When a client sends a schema reference (hash), the server + * can look up the cached schema instead of re-parsing the full schema each time. + *

    + * This is a pure Java implementation of XXHash64 based on the original algorithm + * by Yann Collet. It's optimized for small inputs typical of schema hashing. + * + * @see xxHash + */ +public final class QwpSchemaHash { + + // Default seed (0 for QWP v1) + private static final long DEFAULT_SEED = 0L; + // XXHash64 constants + private static final long PRIME64_1 = 0x9E3779B185EBCA87L; + private static final long PRIME64_2 = 0xC2B2AE3D27D4EB4FL; + // Thread-local Hasher to avoid allocation on every computeSchemaHash call + private static final ThreadLocal HASHER_POOL = ThreadLocal.withInitial(Hasher::new); + private static final long PRIME64_3 = 0x165667B19E3779F9L; + private static final long PRIME64_4 = 0x85EBCA77C2B2AE63L; + private static final long PRIME64_5 = 0x27D4EB2F165667C5L; + + private QwpSchemaHash() { + // utility class + } + + /** + * Computes the schema hash for QWP v1 using String column names. + * Note: Iterates over String chars and converts to UTF-8 bytes directly to avoid getBytes() allocation. + * + * @param columnNames array of column names + * @param columnTypes array of type codes + * @return the schema hash + */ + public static long computeSchemaHash(String[] columnNames, byte[] columnTypes) { + // Use pooled hasher to avoid allocation + Hasher hasher = HASHER_POOL.get(); + hasher.reset(DEFAULT_SEED); + + for (int i = 0; i < columnNames.length; i++) { + String name = columnNames[i]; + // Encode UTF-8 directly without allocating byte array + for (int j = 0, len = name.length(); j < len; j++) { + char c = name.charAt(j); + if (c < 0x80) { + // Single byte (ASCII) + hasher.update((byte) c); + } else if (c < 0x800) { + // Two bytes + hasher.update((byte) (0xC0 | (c >> 6))); + hasher.update((byte) (0x80 | (c & 0x3F))); + } else if (c >= 0xD800 && c <= 0xDBFF && j + 1 < len) { + // Surrogate pair (4 bytes) + char c2 = name.charAt(++j); + if (Character.isLowSurrogate(c2)) { + int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); + hasher.update((byte) (0xF0 | (codePoint >> 18))); + hasher.update((byte) (0x80 | ((codePoint >> 12) & 0x3F))); + hasher.update((byte) (0x80 | ((codePoint >> 6) & 0x3F))); + hasher.update((byte) (0x80 | (codePoint & 0x3F))); + } else { + hasher.update((byte) '?'); + j--; + } + } else if (Character.isSurrogate(c)) { + hasher.update((byte) '?'); + } else { + // Three bytes + hasher.update((byte) (0xE0 | (c >> 12))); + hasher.update((byte) (0x80 | ((c >> 6) & 0x3F))); + hasher.update((byte) (0x80 | (c & 0x3F))); + } + } + hasher.update(columnTypes[i]); + } + + return hasher.getValue(); + } + + /** + * Computes the schema hash directly from column buffers without intermediate arrays. + * This is the most efficient method when column data is already available. + * + * @param columns list of column buffers + * @return the schema hash + */ + public static long computeSchemaHashDirect(io.questdb.client.std.ObjList columns) { + // Use pooled hasher to avoid allocation + Hasher hasher = HASHER_POOL.get(); + hasher.reset(DEFAULT_SEED); + + for (int i = 0, n = columns.size(); i < n; i++) { + QwpTableBuffer.ColumnBuffer col = columns.get(i); + String name = col.getName(); + // Encode UTF-8 directly without allocating byte array + for (int j = 0, len = name.length(); j < len; j++) { + char c = name.charAt(j); + if (c < 0x80) { + hasher.update((byte) c); + } else if (c < 0x800) { + hasher.update((byte) (0xC0 | (c >> 6))); + hasher.update((byte) (0x80 | (c & 0x3F))); + } else if (c >= 0xD800 && c <= 0xDBFF && j + 1 < len) { + char c2 = name.charAt(++j); + if (Character.isLowSurrogate(c2)) { + int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); + hasher.update((byte) (0xF0 | (codePoint >> 18))); + hasher.update((byte) (0x80 | ((codePoint >> 12) & 0x3F))); + hasher.update((byte) (0x80 | ((codePoint >> 6) & 0x3F))); + hasher.update((byte) (0x80 | (codePoint & 0x3F))); + } else { + hasher.update((byte) '?'); + j--; + } + } else if (Character.isSurrogate(c)) { + hasher.update((byte) '?'); + } else { + hasher.update((byte) (0xE0 | (c >> 12))); + hasher.update((byte) (0x80 | ((c >> 6) & 0x3F))); + hasher.update((byte) (0x80 | (c & 0x3F))); + } + } + // Wire type code: just the base type, no nullable flag + hasher.update(col.getType()); + } + + return hasher.getValue(); + } + + /** + * Computes XXHash64 of direct memory with custom seed. + * + * @param address start address + * @param length number of bytes + * @param seed the hash seed + * @return the 64-bit hash value + */ + public static long hash(long address, long length, long seed) { + long h64; + long end = address + length; + long pos = address; + + if (length >= 32) { + long limit = end - 32; + long v1 = seed + PRIME64_1 + PRIME64_2; + long v2 = seed + PRIME64_2; + long v3 = seed; + long v4 = seed - PRIME64_1; + + do { + v1 = round(v1, Unsafe.getUnsafe().getLong(pos)); + pos += 8; + v2 = round(v2, Unsafe.getUnsafe().getLong(pos)); + pos += 8; + v3 = round(v3, Unsafe.getUnsafe().getLong(pos)); + pos += 8; + v4 = round(v4, Unsafe.getUnsafe().getLong(pos)); + pos += 8; + } while (pos <= limit); + + h64 = Long.rotateLeft(v1, 1) + Long.rotateLeft(v2, 7) + + Long.rotateLeft(v3, 12) + Long.rotateLeft(v4, 18); + h64 = mergeRound(h64, v1); + h64 = mergeRound(h64, v2); + h64 = mergeRound(h64, v3); + h64 = mergeRound(h64, v4); + } else { + h64 = seed + PRIME64_5; + } + + h64 += length; + + // Process remaining 8-byte blocks + while (pos + 8 <= end) { + long k1 = Unsafe.getUnsafe().getLong(pos); + k1 *= PRIME64_2; + k1 = Long.rotateLeft(k1, 31); + k1 *= PRIME64_1; + h64 ^= k1; + h64 = Long.rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4; + pos += 8; + } + + // Process remaining 4-byte block + if (pos + 4 <= end) { + h64 ^= (Unsafe.getUnsafe().getInt(pos) & 0xFFFFFFFFL) * PRIME64_1; + h64 = Long.rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3; + pos += 4; + } + + // Process remaining bytes + while (pos < end) { + h64 ^= (Unsafe.getUnsafe().getByte(pos) & 0xFFL) * PRIME64_5; + h64 = Long.rotateLeft(h64, 11) * PRIME64_1; + pos++; + } + + return avalanche(h64); + } + + /** + * Computes XXHash64 of a byte array. + * + * @param data the data to hash + * @return the 64-bit hash value + */ + public static long hash(byte[] data) { + return hash(data, 0, data.length, DEFAULT_SEED); + } + + /** + * Computes XXHash64 of a byte array region with custom seed. + * + * @param data the data to hash + * @param offset starting offset + * @param length number of bytes to hash + * @param seed the hash seed + * @return the 64-bit hash value + */ + public static long hash(byte[] data, int offset, int length, long seed) { + long h64; + int end = offset + length; + int pos = offset; + + if (length >= 32) { + int limit = end - 32; + long v1 = seed + PRIME64_1 + PRIME64_2; + long v2 = seed + PRIME64_2; + long v3 = seed; + long v4 = seed - PRIME64_1; + + do { + v1 = round(v1, getLong(data, pos)); + pos += 8; + v2 = round(v2, getLong(data, pos)); + pos += 8; + v3 = round(v3, getLong(data, pos)); + pos += 8; + v4 = round(v4, getLong(data, pos)); + pos += 8; + } while (pos <= limit); + + h64 = Long.rotateLeft(v1, 1) + Long.rotateLeft(v2, 7) + + Long.rotateLeft(v3, 12) + Long.rotateLeft(v4, 18); + h64 = mergeRound(h64, v1); + h64 = mergeRound(h64, v2); + h64 = mergeRound(h64, v3); + h64 = mergeRound(h64, v4); + } else { + h64 = seed + PRIME64_5; + } + + h64 += length; + + // Process remaining 8-byte blocks + while (pos + 8 <= end) { + long k1 = getLong(data, pos); + k1 *= PRIME64_2; + k1 = Long.rotateLeft(k1, 31); + k1 *= PRIME64_1; + h64 ^= k1; + h64 = Long.rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4; + pos += 8; + } + + // Process remaining 4-byte block + if (pos + 4 <= end) { + h64 ^= (getInt(data, pos) & 0xFFFFFFFFL) * PRIME64_1; + h64 = Long.rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3; + pos += 4; + } + + // Process remaining bytes + while (pos < end) { + h64 ^= (data[pos] & 0xFFL) * PRIME64_5; + h64 = Long.rotateLeft(h64, 11) * PRIME64_1; + pos++; + } + + return avalanche(h64); + } + + /** + * Computes XXHash64 of direct memory. + * + * @param address start address + * @param length number of bytes + * @return the 64-bit hash value + */ + public static long hash(long address, long length) { + return hash(address, length, DEFAULT_SEED); + } + + private static long avalanche(long h64) { + h64 ^= h64 >>> 33; + h64 *= PRIME64_2; + h64 ^= h64 >>> 29; + h64 *= PRIME64_3; + h64 ^= h64 >>> 32; + return h64; + } + + private static int getInt(byte[] data, int pos) { + return (data[pos] & 0xFF) | + ((data[pos + 1] & 0xFF) << 8) | + ((data[pos + 2] & 0xFF) << 16) | + ((data[pos + 3] & 0xFF) << 24); + } + + private static long getLong(byte[] data, int pos) { + return ((long) data[pos] & 0xFF) | + (((long) data[pos + 1] & 0xFF) << 8) | + (((long) data[pos + 2] & 0xFF) << 16) | + (((long) data[pos + 3] & 0xFF) << 24) | + (((long) data[pos + 4] & 0xFF) << 32) | + (((long) data[pos + 5] & 0xFF) << 40) | + (((long) data[pos + 6] & 0xFF) << 48) | + (((long) data[pos + 7] & 0xFF) << 56); + } + + private static long mergeRound(long acc, long val) { + val = round(0, val); + acc ^= val; + acc = acc * PRIME64_1 + PRIME64_4; + return acc; + } + + private static long round(long acc, long input) { + acc += input * PRIME64_2; + acc = Long.rotateLeft(acc, 31); + acc *= PRIME64_1; + return acc; + } + + /** + * Streaming hasher for incremental hash computation. + *

    + * This is useful when building the schema hash incrementally + * as columns are processed. + */ + public static class Hasher { + private final byte[] buffer = new byte[32]; + private int bufferPos; + private long seed; + private long totalLen; + private long v1, v2, v3, v4; + + public Hasher() { + reset(DEFAULT_SEED); + } + + /** + * Finalizes and returns the hash value. + * + * @return the 64-bit hash + */ + public long getValue() { + long h64; + + if (totalLen >= 32) { + h64 = Long.rotateLeft(v1, 1) + Long.rotateLeft(v2, 7) + + Long.rotateLeft(v3, 12) + Long.rotateLeft(v4, 18); + h64 = mergeRound(h64, v1); + h64 = mergeRound(h64, v2); + h64 = mergeRound(h64, v3); + h64 = mergeRound(h64, v4); + } else { + h64 = seed + PRIME64_5; + } + + h64 += totalLen; + + // Process buffered data + int pos = 0; + while (pos + 8 <= bufferPos) { + long k1 = getLong(buffer, pos); + k1 *= PRIME64_2; + k1 = Long.rotateLeft(k1, 31); + k1 *= PRIME64_1; + h64 ^= k1; + h64 = Long.rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4; + pos += 8; + } + + if (pos + 4 <= bufferPos) { + h64 ^= (getInt(buffer, pos) & 0xFFFFFFFFL) * PRIME64_1; + h64 = Long.rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3; + pos += 4; + } + + while (pos < bufferPos) { + h64 ^= (buffer[pos] & 0xFFL) * PRIME64_5; + h64 = Long.rotateLeft(h64, 11) * PRIME64_1; + pos++; + } + + return avalanche(h64); + } + + /** + * Resets the hasher with the given seed. + * + * @param seed the hash seed + */ + public void reset(long seed) { + this.seed = seed; + v1 = seed + PRIME64_1 + PRIME64_2; + v2 = seed + PRIME64_2; + v3 = seed; + v4 = seed - PRIME64_1; + totalLen = 0; + bufferPos = 0; + } + + /** + * Updates the hash with a byte array. + * + * @param data the bytes to add + */ + public void update(byte[] data) { + update(data, 0, data.length); + } + + /** + * Updates the hash with a byte array region. + * + * @param data the bytes to add + * @param offset starting offset + * @param length number of bytes + */ + public void update(byte[] data, int offset, int length) { + totalLen += length; + + // Fill buffer first + if (bufferPos > 0) { + int toCopy = Math.min(32 - bufferPos, length); + System.arraycopy(data, offset, buffer, bufferPos, toCopy); + bufferPos += toCopy; + offset += toCopy; + length -= toCopy; + + if (bufferPos == 32) { + processBuffer(); + } + } + + // Process 32-byte blocks directly + while (length >= 32) { + v1 = round(v1, getLong(data, offset)); + v2 = round(v2, getLong(data, offset + 8)); + v3 = round(v3, getLong(data, offset + 16)); + v4 = round(v4, getLong(data, offset + 24)); + offset += 32; + length -= 32; + } + + // Buffer remaining + if (length > 0) { + System.arraycopy(data, offset, buffer, 0, length); + bufferPos = length; + } + } + + /** + * Updates the hash with a single byte. + * + * @param b the byte to add + */ + public void update(byte b) { + buffer[bufferPos++] = b; + totalLen++; + + if (bufferPos == 32) { + processBuffer(); + } + } + + private void processBuffer() { + v1 = round(v1, getLong(buffer, 0)); + v2 = round(v2, getLong(buffer, 8)); + v3 = round(v3, getLong(buffer, 16)); + v4 = round(v4, getLong(buffer, 24)); + bufferPos = 0; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java new file mode 100644 index 0000000..9d01120 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -0,0 +1,1820 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.protocol; + +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cairo.TableUtils; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.line.array.ArrayBufferAppender; +import io.questdb.client.cutlass.line.array.DoubleArray; +import io.questdb.client.cutlass.line.array.LongArray; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.std.CharSequenceIntHashMap; +import io.questdb.client.std.Chars; +import io.questdb.client.std.Decimal128; +import io.questdb.client.std.Decimal256; +import io.questdb.client.std.Decimal64; +import io.questdb.client.std.Decimals; +import io.questdb.client.std.LowerCaseAsciiCharSequenceIntHashMap; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.NumericException; +import io.questdb.client.std.ObjList; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.Vect; +import io.questdb.client.std.str.StringSink; +import io.questdb.client.std.str.Utf8s; + +import java.util.Arrays; + +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; + +/** + * Buffers rows for a single table in columnar format. + *

    + * Fixed-width column data is stored off-heap via {@link OffHeapAppendMemory} for zero-GC + * buffering and bulk copy to network buffers. Variable-width data (strings, symbol + * dictionaries, arrays) remains on-heap. + */ +public class QwpTableBuffer implements QuietCloseable { + + private static final int MAX_COLUMN_NAME_LENGTH = 127; + private final LowerCaseAsciiCharSequenceIntHashMap columnNameToIndex; + private final ObjList columns; + private final QwpWebSocketSender sender; + private final String tableName; + private QwpColumnDef[] cachedColumnDefs; + private int columnAccessCursor; // tracks expected next column index + private boolean columnDefsCacheValid; + private int committedColumnCount; // columns that existed at last nextRow() + private ColumnBuffer[] fastColumns; // plain array for O(1) sequential access + private int rowCount; + private long schemaHash; + private boolean schemaHashComputed; + + public QwpTableBuffer(String tableName) { + this(tableName, null); + } + + /** + * Use this constructor overload to allow writing to a symbol column. + * {@link ColumnBuffer#addSymbol(CharSequence)} needs the sender to + * call {@link QwpWebSocketSender#getOrAddGlobalSymbol(CharSequence)}, registering + * the symbol in the global dictionary shared with the server. + */ + public QwpTableBuffer(String tableName, QwpWebSocketSender sender) { + this.tableName = tableName; + this.sender = sender; + this.columns = new ObjList<>(); + this.columnNameToIndex = new LowerCaseAsciiCharSequenceIntHashMap(); + this.rowCount = 0; + this.schemaHash = 0; + this.schemaHashComputed = false; + this.columnDefsCacheValid = false; + } + + /** + * Cancels the current in-progress row. + *

    + * This removes any column values added since the last {@link #nextRow()} call. + * If no values have been added for the current row, this is a no-op. + */ + public void cancelCurrentRow() { + columnAccessCursor = 0; + for (int i = 0, n = columns.size(); i < n; i++) { + ColumnBuffer col = fastColumns[i]; + if (i >= committedColumnCount) { + // Column was created during the in-progress row. Remove all data. + col.truncateTo(0); + } else if (col.size > rowCount) { + // Pre-existing column was set for the in-progress row. + // Truncate to committed state. + col.truncateTo(rowCount); + } + // else: pre-existing column wasn't touched this row. No-op. + } + } + + /** + * Clears the buffer completely, including column definitions. + * Frees all off-heap memory. + */ + public void clear() { + for (int i = 0, n = columns.size(); i < n; i++) { + columns.get(i).close(); + } + columns.clear(); + columnNameToIndex.clear(); + fastColumns = null; + columnAccessCursor = 0; + committedColumnCount = 0; + rowCount = 0; + schemaHash = 0; + schemaHashComputed = false; + columnDefsCacheValid = false; + cachedColumnDefs = null; + } + + @Override + public void close() { + clear(); + } + + /** + * Returns the total bytes buffered across all columns. + * This queries actual buffer sizes, not estimates. + */ + public long getBufferedBytes() { + long bytes = 0; + for (int i = 0, n = columns.size(); i < n; i++) { + bytes += fastColumns[i].getBufferedBytes(); + } + return bytes; + } + + /** + * Returns the column at the given index. + */ + public ColumnBuffer getColumn(int index) { + return columns.get(index); + } + + /** + * Returns the number of columns. + */ + public int getColumnCount() { + return columns.size(); + } + + /** + * Returns the column definitions (cached for efficiency). + */ + public QwpColumnDef[] getColumnDefs() { + if (!columnDefsCacheValid || cachedColumnDefs == null || cachedColumnDefs.length != columns.size()) { + cachedColumnDefs = new QwpColumnDef[columns.size()]; + for (int i = 0; i < columns.size(); i++) { + ColumnBuffer col = columns.get(i); + cachedColumnDefs[i] = new QwpColumnDef(col.name, col.type); + } + columnDefsCacheValid = true; + } + return cachedColumnDefs; + } + + /** + * Returns an existing column with the given name and type, or {@code null} if absent. + *

    + * Uses the same sequential access optimization as {@link #getOrCreateColumn(CharSequence, byte, boolean)}. + * When the next expected column is accessed in order, the internal cursor advances without a hash lookup. + */ + public ColumnBuffer getExistingColumn(CharSequence name, byte type) { + return lookupColumn(name, type); + } + + /** + * Gets or creates a column with the given name and type. + *

    + * Optimized for the common case where columns are accessed in the same + * order every row: a sequential cursor avoids hash map lookups entirely. + *

    + * Returns {@code null} when the column has already been written in the current + * (uncommitted) row. Callers must treat a {@code null} return as "duplicate + * column in this row — skip the write", matching the ILP first-value-wins + * semantics. The check is a single field comparison on the hot path and has + * no measurable cost. + */ + public ColumnBuffer getOrCreateColumn(CharSequence name, byte type, boolean useNullBitmap) { + if (name == null || name.length() == 0) { + throw new LineSenderException("column name cannot be empty"); + } + ColumnBuffer existing = lookupColumn(name, type); + if (existing != null) { + // col.size > rowCount means this column already received a value + // for the in-progress row. Silently ignore the duplicate (first + // value wins, same as the ILP server behaviour). + return existing.size <= rowCount ? existing : null; + } + if (TableUtils.isValidColumnName(name, MAX_COLUMN_NAME_LENGTH)) { + return createColumn(name, type, useNullBitmap); + } + throw new LineSenderException( + name.length() > MAX_COLUMN_NAME_LENGTH ? "column name too long [maxLength=" + MAX_COLUMN_NAME_LENGTH + "]" + : "column name contains illegal characters: " + name + ); + } + + public ColumnBuffer getOrCreateDesignatedTimestampColumn(byte type) { + ColumnBuffer existing = lookupColumn("", type); + if (existing != null) { + return existing; + } + return createColumn("", type, true); + } + + /** + * Returns the number of rows buffered. + */ + public int getRowCount() { + return rowCount; + } + + /** + * Returns the schema hash for this table. + *

    + * The hash is computed to match what QwpSchema.computeSchemaHash() produces: + * - Uses wire type codes (with nullable bit) + * - Hash is over name bytes + type code for each column + */ + public long getSchemaHash() { + if (!schemaHashComputed) { + // Compute hash directly from column buffers without intermediate arrays + schemaHash = QwpSchemaHash.computeSchemaHashDirect(columns); + schemaHashComputed = true; + } + return schemaHash; + } + + /** + * Returns the table name. + */ + public String getTableName() { + return tableName; + } + + /** + * Advances to the next row. + *

    + * This should be called after all column values for the current row have been set. + */ + public void nextRow() { + // Reset sequential access cursor for the next row + columnAccessCursor = 0; + // Ensure all columns have the same row count + for (int i = 0, n = columns.size(); i < n; i++) { + ColumnBuffer col = fastColumns[i]; + // If column wasn't set for this row, add a null + while (col.size < rowCount + 1) { + col.addNull(); + } + } + rowCount++; + committedColumnCount = columns.size(); + } + + /** + * Advances to the next row using a prepared list of columns that need null padding. + *

    + * This avoids rescanning every column when the caller has already identified + * which columns were omitted in the current row. + */ + public void nextRow(ColumnBuffer[] missingColumns, int missingColumnCount) { + columnAccessCursor = 0; + for (int i = 0; i < missingColumnCount; i++) { + ColumnBuffer col = missingColumns[i]; + while (col.size < rowCount + 1) { + col.addNull(); + } + } + rowCount++; + committedColumnCount = columns.size(); + } + + /** + * Resets the buffer for reuse. Keeps column definitions and allocated memory. + */ + public void reset() { + for (int i = 0, n = columns.size(); i < n; i++) { + fastColumns[i].reset(); + } + columnAccessCursor = 0; + committedColumnCount = columns.size(); + rowCount = 0; + } + + public void retainInProgressRow( + int[] sizeBefore, + int[] valueCountBefore, + int[] arrayShapeOffsetBefore, + int[] arrayDataOffsetBefore + ) { + columnAccessCursor = 0; + for (int i = 0, n = columns.size(); i < n; i++) { + ColumnBuffer col = fastColumns[i]; + if (sizeBefore[i] > -1) { + col.retainTailRow( + sizeBefore[i], + valueCountBefore[i], + arrayShapeOffsetBefore[i], + arrayDataOffsetBefore[i] + ); + } else { + col.clearToEmptyFast(); + } + } + rowCount = 0; + committedColumnCount = columns.size(); + } + + public void rollbackUncommittedColumns() { + if (columns.size() <= committedColumnCount) { + return; + } + + for (int i = columns.size() - 1; i >= committedColumnCount; i--) { + ColumnBuffer col = columns.getQuick(i); + if (col != null) { + col.close(); + } + columns.remove(i); + } + rebuildColumnAccessStructures(); + } + + private static void assertColumnType(CharSequence name, byte type, ColumnBuffer column) { + if (column.type != type) { + throw new LineSenderException( + "Column type mismatch for column '" + name + "': columnType=" + + column.type + ", sentType=" + type + ); + } + } + + private ColumnBuffer createColumn(CharSequence name, byte type, boolean useNullBitmap) { + ColumnBuffer col = new ColumnBuffer(Chars.toString(name), type, useNullBitmap); + col.sender = sender; + int index = columns.size(); + col.index = index; + columns.add(col); + columnNameToIndex.put(name, index); + // Update fast access array + if (fastColumns == null || index >= fastColumns.length) { + int newLen = Math.max(8, index + 4); + ColumnBuffer[] newArr = new ColumnBuffer[newLen]; + if (fastColumns != null) { + System.arraycopy(fastColumns, 0, newArr, 0, index); + } + fastColumns = newArr; + } + fastColumns[index] = col; + // Pre-pad with nulls for already-committed rows so the next + // value the caller adds lands at the correct row position. + for (int r = 0; r < rowCount; r++) { + col.addNull(); + } + schemaHashComputed = false; + columnDefsCacheValid = false; + return col; + } + + private ColumnBuffer lookupColumn(CharSequence name, byte type) { + // Fast path: predict next column in sequence + int n = columns.size(); + if (columnAccessCursor < n) { + ColumnBuffer candidate = fastColumns[columnAccessCursor]; + if (Chars.equalsIgnoreCase(candidate.name, name)) { + columnAccessCursor++; + assertColumnType(name, type, candidate); + return candidate; + } + } + + // Slow path: hash map lookup + int idx = columnNameToIndex.get(name); + if (idx >= 0) { + ColumnBuffer existing = columns.get(idx); + assertColumnType(name, type, existing); + return existing; + } + + return null; + } + + private void rebuildColumnAccessStructures() { + columnNameToIndex.clear(); + + int columnCount = columns.size(); + int minCapacity = Math.max(8, columnCount + 4); + if (fastColumns == null || fastColumns.length < minCapacity) { + fastColumns = new ColumnBuffer[minCapacity]; + } else { + Arrays.fill(fastColumns, null); + } + + for (int i = 0; i < columnCount; i++) { + ColumnBuffer col = columns.getQuick(i); + col.index = i; + fastColumns[i] = col; + columnNameToIndex.put(col.name, i); + } + + schemaHashComputed = false; + columnDefsCacheValid = false; + cachedColumnDefs = null; + } + + /** + * Returns the in-memory buffer element stride in bytes. This is the size used + * to store each value in the client's off-heap {@link OffHeapAppendMemory} buffer. + * This is different from element size on the wire. + *

    + * For example, BOOLEAN is stored as 1 byte per value here (for easy indexed access) + * but bit-packed on the wire; GEOHASH is stored as 8-byte longs here but uses + * variable-width encoding on the wire. + *

    + * Returns 0 for variable-width types (string, arrays) that do not use a fixed-stride + * data buffer. + * + * @see QwpConstants#getFixedTypeSize(byte) for wire-format sizes + */ + static int elementSizeInBuffer(byte type) { + switch (type) { + case TYPE_BOOLEAN: + case TYPE_BYTE: + return 1; + case TYPE_SHORT: + case TYPE_CHAR: + return 2; + case TYPE_INT: + case TYPE_SYMBOL: + case TYPE_FLOAT: + return 4; + case TYPE_GEOHASH: + case TYPE_LONG: + case TYPE_TIMESTAMP: + case TYPE_TIMESTAMP_NANOS: + case TYPE_DATE: + case TYPE_DECIMAL64: + case TYPE_DOUBLE: + return 8; + case TYPE_UUID: + case TYPE_DECIMAL128: + return 16; + case TYPE_LONG256: + case TYPE_DECIMAL256: + return 32; + default: + return 0; + } + } + + /** + * Helper class to capture array data from DoubleArray/LongArray.appendToBufPtr(). + */ + private static class ArrayCapture implements ArrayBufferAppender { + final int[] shape = new int[32]; + double[] doubleData; + int doubleDataOffset; + long[] longData; + int longDataOffset; + byte nDims; + private boolean forLong; + private int shapeIndex; + + @Override + public void putBlockOfBytes(long from, long len) { + int count = (int) (len / 8); + if (forLong) { + if (longData == null || longData.length < count) { + longData = new long[count]; + } + for (int i = 0; i < count; i++) { + longData[longDataOffset++] = Unsafe.getUnsafe().getLong(from + i * 8L); + } + } else { + if (doubleData == null || doubleData.length < count) { + doubleData = new double[count]; + } + for (int i = 0; i < count; i++) { + doubleData[doubleDataOffset++] = Unsafe.getUnsafe().getDouble(from + i * 8L); + } + } + } + + @Override + public void putByte(byte b) { + if (shapeIndex == 0) { + nDims = b; + } + } + + @Override + public void putDouble(double value) { + if (doubleData != null && doubleDataOffset < doubleData.length) { + doubleData[doubleDataOffset++] = value; + } + } + + @Override + public void putInt(int value) { + if (shapeIndex < nDims) { + shape[shapeIndex++] = value; + if (shapeIndex == nDims) { + int totalElements = 1; + for (int i = 0; i < nDims; i++) { + totalElements *= shape[i]; + } + if (forLong) { + if (longData == null || longData.length < totalElements) { + longData = new long[totalElements]; + } + } else { + if (doubleData == null || doubleData.length < totalElements) { + doubleData = new double[totalElements]; + } + } + } + } + } + + @Override + public void putLong(long value) { + if (longData != null && longDataOffset < longData.length) { + longData[longDataOffset++] = value; + } + } + + void reset(boolean forLong) { + this.forLong = forLong; + shapeIndex = 0; + nDims = 0; + doubleDataOffset = 0; + longDataOffset = 0; + } + } + + /** + * Column buffer for a single column. + *

    + * Fixed-width data is stored off-heap in {@link OffHeapAppendMemory} for zero-GC + * operation and efficient bulk copy to network buffers. + */ + public static class ColumnBuffer implements QuietCloseable { + private static final long DOUBLE_ARRAY_BASE_OFFSET = Unsafe.getUnsafe().arrayBaseOffset(double[].class); + final int elemSize; + final String name; + final byte type; + final boolean useNullBitmap; + private final Decimal256 rescaleTemp = new Decimal256(); + private ArrayCapture arrayCapture; + private int arrayDataOffset; + // Array storage (double/long arrays - variable length per row) + private byte[] arrayDims; + private int arrayShapeOffset; + private int[] arrayShapes; + // Optional auxiliary buffer used by symbol encoders that need sideband IDs. + private OffHeapAppendMemory auxBuffer; + // Off-heap data buffer for fixed-width types + private OffHeapAppendMemory dataBuffer; + // Decimal storage + private byte decimalScale = -1; + private double[] doubleArrayData; + // GeoHash precision (number of bits, 1-60) + private int geohashPrecision = -1; + private boolean hasNulls; + private int index; + private long[] longArrayData; + private int maxGlobalSymbolId = -1; + private int nullBufCapRows; + // Off-heap null bitmap (bit-packed, 1 bit per row) + private long nullBufPtr; + private QwpWebSocketSender sender; + private int size; // Total row count (including nulls) + // Symbol specific (dictionary stays on-heap) + private boolean storeGlobalSymbolIdsOnly; + private OffHeapAppendMemory stringData; + // Off-heap storage for string/varchar column data + private OffHeapAppendMemory stringOffsets; + private CharSequenceIntHashMap symbolDict; + private ObjList symbolList; + private StringSink symbolLookupSink; + private int valueCount; // Actual stored values (excludes nulls) + + public ColumnBuffer(String name, byte type, boolean useNullBitmap) { + this.name = name; + this.type = type; + this.useNullBitmap = useNullBitmap; + this.elemSize = elementSizeInBuffer(type); + this.size = 0; + this.valueCount = 0; + this.hasNulls = false; + + try { + allocateStorage(type); + if (useNullBitmap) { + nullBufCapRows = 64; // multiple of 64 + long sizeBytes = (long) nullBufCapRows >>> 3; + nullBufPtr = Unsafe.calloc(sizeBytes, MemoryTag.NATIVE_ILP_RSS); + } + } catch (Throwable t) { + close(); + throw t; + } + } + + public void addBoolean(boolean value) { + dataBuffer.putByte(value ? (byte) 1 : (byte) 0); + valueCount++; + size++; + } + + public void addByte(byte value) { + dataBuffer.putByte(value); + valueCount++; + size++; + } + + public void addDecimal128(Decimal128 value) { + if (value == null || value.isNull()) { + addNull(); + return; + } + if (decimalScale == -1) { + decimalScale = (byte) value.getScale(); + } else if (decimalScale != value.getScale()) { + rescaleTemp.ofRaw(value.getHigh(), value.getLow()); + rescaleTemp.setScale(value.getScale()); + try { + rescaleTemp.rescale(decimalScale); + } catch (NumericException e) { + throw new LineSenderException("column '" + name + "' cannot rescale decimal from scale " + + value.getScale() + " to " + decimalScale + " without precision loss", e); + } + if (!rescaleTemp.fitsInStorageSizePow2(4)) { + throw new LineSenderException("Decimal128 overflow: rescaling from scale " + + value.getScale() + " to " + decimalScale + " exceeds 128-bit capacity"); + } + dataBuffer.putLong(rescaleTemp.getLh()); + dataBuffer.putLong(rescaleTemp.getLl()); + valueCount++; + size++; + return; + } + dataBuffer.putLong(value.getHigh()); + dataBuffer.putLong(value.getLow()); + valueCount++; + size++; + } + + public void addDecimal256(Decimal256 value) { + if (value == null || value.isNull()) { + addNull(); + return; + } + Decimal256 src = value; + if (decimalScale == -1) { + decimalScale = (byte) value.getScale(); + } else if (decimalScale != value.getScale()) { + rescaleTemp.copyFrom(value); + try { + rescaleTemp.rescale(decimalScale); + } catch (NumericException e) { + throw new LineSenderException("column '" + name + "' cannot rescale decimal from scale " + + value.getScale() + " to " + decimalScale + " without precision loss", e); + } + src = rescaleTemp; + } + dataBuffer.putLong(src.getHh()); + dataBuffer.putLong(src.getHl()); + dataBuffer.putLong(src.getLh()); + dataBuffer.putLong(src.getLl()); + valueCount++; + size++; + } + + public void addDecimal64(Decimal64 value) { + if (value == null || value.isNull()) { + addNull(); + return; + } + if (decimalScale == -1) { + decimalScale = (byte) value.getScale(); + dataBuffer.putLong(value.getValue()); + } else if (decimalScale != value.getScale()) { + rescaleTemp.ofRaw(value.getValue()); + rescaleTemp.setScale(value.getScale()); + try { + rescaleTemp.rescale(decimalScale); + } catch (NumericException e) { + throw new LineSenderException("column '" + name + "' cannot rescale decimal from scale " + + value.getScale() + " to " + decimalScale + " without precision loss", e); + } + if (!rescaleTemp.fitsInStorageSizePow2(3)) { + throw new LineSenderException("Decimal64 overflow: rescaling from scale " + + value.getScale() + " to " + decimalScale + " exceeds 64-bit capacity"); + } + dataBuffer.putLong(rescaleTemp.getLl()); + } else { + dataBuffer.putLong(value.getValue()); + } + valueCount++; + size++; + } + + public void addDouble(double value) { + dataBuffer.putDouble(value); + valueCount++; + size++; + } + + public void addDoubleArray(double[] values) { + if (values == null) { + addNull(); + return; + } + ensureArrayCapacity(1, values.length); + arrayDims[valueCount] = 1; + arrayShapes[arrayShapeOffset++] = values.length; + for (double v : values) { + doubleArrayData[arrayDataOffset++] = v; + } + valueCount++; + size++; + } + + public void addDoubleArray(double[][] values) { + if (values == null) { + addNull(); + return; + } + int dim0 = values.length; + int dim1 = dim0 > 0 ? values[0].length : 0; + for (int i = 1; i < dim0; i++) { + if (values[i].length != dim1) { + throw new LineSenderException("irregular array shape"); + } + } + int elemCount = checkedElementCount((long) dim0 * dim1); + ensureArrayCapacity(2, elemCount); + arrayDims[valueCount] = 2; + arrayShapes[arrayShapeOffset++] = dim0; + arrayShapes[arrayShapeOffset++] = dim1; + for (double[] row : values) { + for (double v : row) { + doubleArrayData[arrayDataOffset++] = v; + } + } + valueCount++; + size++; + } + + public void addDoubleArray(double[][][] values) { + if (values == null) { + addNull(); + return; + } + int dim0 = values.length; + int dim1 = dim0 > 0 ? values[0].length : 0; + int dim2 = dim0 > 0 && dim1 > 0 ? values[0][0].length : 0; + for (int i = 0; i < dim0; i++) { + if (values[i].length != dim1) { + throw new LineSenderException("irregular array shape"); + } + for (int j = 0; j < dim1; j++) { + if (values[i][j].length != dim2) { + throw new LineSenderException("irregular array shape"); + } + } + } + int elemCount = checkedElementCount((long) dim0 * dim1 * dim2); + ensureArrayCapacity(3, elemCount); + arrayDims[valueCount] = 3; + arrayShapes[arrayShapeOffset++] = dim0; + arrayShapes[arrayShapeOffset++] = dim1; + arrayShapes[arrayShapeOffset++] = dim2; + for (double[][] plane : values) { + for (double[] row : plane) { + for (double v : row) { + doubleArrayData[arrayDataOffset++] = v; + } + } + } + valueCount++; + size++; + } + + public void addDoubleArray(DoubleArray array) { + if (array == null) { + addNull(); + return; + } + arrayCapture.reset(false); + array.appendToBufPtr(arrayCapture); + + ensureArrayCapacity(arrayCapture.nDims, arrayCapture.doubleDataOffset); + arrayDims[valueCount] = arrayCapture.nDims; + for (int i = 0; i < arrayCapture.nDims; i++) { + arrayShapes[arrayShapeOffset++] = arrayCapture.shape[i]; + } + for (int i = 0; i < arrayCapture.doubleDataOffset; i++) { + doubleArrayData[arrayDataOffset++] = arrayCapture.doubleData[i]; + } + valueCount++; + size++; + } + + public void addDoubleArrayPayload(long ptr, long len) { + appendArrayPayload(ptr, len); + } + + public void addFloat(float value) { + dataBuffer.putFloat(value); + valueCount++; + size++; + } + + /** + * Adds a geohash value with the given precision. + * + * @param value the geohash value (bit-packed) + * @param precision number of bits (1-60) + */ + public void addGeoHash(long value, int precision) { + if (precision < 1 || precision > 60) { + throw new LineSenderException("invalid GeoHash precision: " + precision + " (must be 1-60)"); + } + if (geohashPrecision == -1) { + geohashPrecision = precision; + } else if (geohashPrecision != precision) { + throw new LineSenderException( + "GeoHash precision mismatch: column has " + geohashPrecision + " bits, got " + precision + ); + } + dataBuffer.putLong(value); + valueCount++; + size++; + } + + public void addInt(int value) { + dataBuffer.putInt(value); + valueCount++; + size++; + } + + public void addLong(long value) { + dataBuffer.putLong(value); + valueCount++; + size++; + } + + public void addLong256(long l0, long l1, long l2, long l3) { + dataBuffer.putLong(l0); + dataBuffer.putLong(l1); + dataBuffer.putLong(l2); + dataBuffer.putLong(l3); + valueCount++; + size++; + } + + public void addLongArray(long[] values) { + if (values == null) { + addNull(); + return; + } + ensureArrayCapacity(1, values.length); + arrayDims[valueCount] = 1; + arrayShapes[arrayShapeOffset++] = values.length; + for (long v : values) { + longArrayData[arrayDataOffset++] = v; + } + valueCount++; + size++; + } + + public void addLongArray(long[][] values) { + if (values == null) { + addNull(); + return; + } + int dim0 = values.length; + int dim1 = dim0 > 0 ? values[0].length : 0; + for (int i = 1; i < dim0; i++) { + if (values[i].length != dim1) { + throw new LineSenderException("irregular array shape"); + } + } + int elemCount = checkedElementCount((long) dim0 * dim1); + ensureArrayCapacity(2, elemCount); + arrayDims[valueCount] = 2; + arrayShapes[arrayShapeOffset++] = dim0; + arrayShapes[arrayShapeOffset++] = dim1; + for (long[] row : values) { + for (long v : row) { + longArrayData[arrayDataOffset++] = v; + } + } + valueCount++; + size++; + } + + public void addLongArray(long[][][] values) { + if (values == null) { + addNull(); + return; + } + int dim0 = values.length; + int dim1 = dim0 > 0 ? values[0].length : 0; + int dim2 = dim0 > 0 && dim1 > 0 ? values[0][0].length : 0; + for (int i = 0; i < dim0; i++) { + if (values[i].length != dim1) { + throw new LineSenderException("irregular array shape"); + } + for (int j = 0; j < dim1; j++) { + if (values[i][j].length != dim2) { + throw new LineSenderException("irregular array shape"); + } + } + } + int elemCount = checkedElementCount((long) dim0 * dim1 * dim2); + ensureArrayCapacity(3, elemCount); + arrayDims[valueCount] = 3; + arrayShapes[arrayShapeOffset++] = dim0; + arrayShapes[arrayShapeOffset++] = dim1; + arrayShapes[arrayShapeOffset++] = dim2; + for (long[][] plane : values) { + for (long[] row : plane) { + for (long v : row) { + longArrayData[arrayDataOffset++] = v; + } + } + } + valueCount++; + size++; + } + + public void addLongArray(LongArray array) { + if (array == null) { + addNull(); + return; + } + arrayCapture.reset(true); + array.appendToBufPtr(arrayCapture); + + ensureArrayCapacity(arrayCapture.nDims, arrayCapture.longDataOffset); + arrayDims[valueCount] = arrayCapture.nDims; + for (int i = 0; i < arrayCapture.nDims; i++) { + arrayShapes[arrayShapeOffset++] = arrayCapture.shape[i]; + } + for (int i = 0; i < arrayCapture.longDataOffset; i++) { + longArrayData[arrayDataOffset++] = arrayCapture.longData[i]; + } + valueCount++; + size++; + } + + public void addNull() { + if (useNullBitmap) { + ensureNullBitmapCapacity(size + 1); + markNull(size); + } else { + // For non-nullable columns, store a sentinel/default value + switch (type) { + case TYPE_BOOLEAN: + case TYPE_BYTE: + dataBuffer.putByte((byte) 0); + break; + case TYPE_SHORT: + case TYPE_CHAR: + dataBuffer.putShort((short) 0); + break; + case TYPE_INT: + dataBuffer.putInt(0); + break; + case TYPE_GEOHASH: + dataBuffer.putLong(-1L); + break; + case TYPE_LONG: + case TYPE_TIMESTAMP: + case TYPE_TIMESTAMP_NANOS: + case TYPE_DATE: + dataBuffer.putLong(Long.MIN_VALUE); + break; + case TYPE_FLOAT: + dataBuffer.putFloat(Float.NaN); + break; + case TYPE_DOUBLE: + dataBuffer.putDouble(Double.NaN); + break; + case TYPE_STRING: + case TYPE_VARCHAR: + stringOffsets.putInt((int) stringData.getAppendOffset()); + break; + case TYPE_SYMBOL: + dataBuffer.putInt(-1); + break; + case TYPE_UUID: + dataBuffer.putLong(Long.MIN_VALUE); + dataBuffer.putLong(Long.MIN_VALUE); + break; + case TYPE_LONG256: + dataBuffer.putLong(Long.MIN_VALUE); + dataBuffer.putLong(Long.MIN_VALUE); + dataBuffer.putLong(Long.MIN_VALUE); + dataBuffer.putLong(Long.MIN_VALUE); + break; + case TYPE_DECIMAL64: + dataBuffer.putLong(Decimals.DECIMAL64_NULL); + break; + case TYPE_DECIMAL128: + dataBuffer.putLong(Decimals.DECIMAL128_HI_NULL); + dataBuffer.putLong(Decimals.DECIMAL128_LO_NULL); + break; + case TYPE_DECIMAL256: + dataBuffer.putLong(Decimals.DECIMAL256_HH_NULL); + dataBuffer.putLong(Decimals.DECIMAL256_HL_NULL); + dataBuffer.putLong(Decimals.DECIMAL256_LH_NULL); + dataBuffer.putLong(Decimals.DECIMAL256_LL_NULL); + break; + case TYPE_DOUBLE_ARRAY: + case TYPE_LONG_ARRAY: + ensureArrayCapacity(1, 0); + arrayDims[valueCount] = 1; + arrayShapes[arrayShapeOffset++] = 0; + break; + } + valueCount++; + } + size++; + } + + public void addShort(short value) { + dataBuffer.putShort(value); + valueCount++; + size++; + } + + public void addString(CharSequence value) { + if (value == null && useNullBitmap) { + ensureNullBitmapCapacity(size + 1); + markNull(size); + } else { + if (value != null) { + stringData.putUtf8(value); + } + stringOffsets.putInt((int) stringData.getAppendOffset()); + valueCount++; + } + size++; + } + + public void addSymbol(CharSequence value) { + if (value == null) { + addNull(); + return; + } + if (sender != null) { + int globalId = sender.getOrAddGlobalSymbol(value); + addSymbolWithGlobalId(value, globalId); + return; + } + if (storeGlobalSymbolIdsOnly) { + throw new LineSenderException("column '" + name + "' cannot mix global symbol IDs with local symbol dictionary values"); + } + int idx = getOrAddLocalSymbol(value); + dataBuffer.putInt(idx); + valueCount++; + size++; + } + + public void addSymbolUtf8(long ptr, int len) { + if (len < 0) { + addNull(); + return; + } + StringSink lookupSink = symbolLookupSink; + if (lookupSink == null) { + symbolLookupSink = lookupSink = new StringSink(Math.max(16, len)); + } else { + lookupSink.clear(); + } + if (!Utf8s.utf8ToUtf16(ptr, ptr + len, lookupSink)) { + // Reuse the existing error path with the same diagnostic payload. + Utf8s.stringFromUtf8Bytes(ptr, ptr + len); + throw new AssertionError("unreachable"); + } + if (sender != null) { + int globalId = sender.getOrAddGlobalSymbol(lookupSink); + addSymbolWithGlobalId(lookupSink, globalId); + return; + } + if (storeGlobalSymbolIdsOnly) { + throw new LineSenderException("column '" + name + "' cannot mix global symbol IDs with local symbol dictionary values"); + } + int idx = getOrAddLocalSymbol(lookupSink); + dataBuffer.putInt(idx); + valueCount++; + size++; + } + + public void addSymbolWithGlobalId(CharSequence value, int globalId) { + if (value == null) { + addNull(); + return; + } + if (!storeGlobalSymbolIdsOnly) { + if (symbolList != null && symbolList.size() > 0) { + int localIdx = getOrAddLocalSymbol(value); + dataBuffer.putInt(localIdx); + if (auxBuffer == null) { + auxBuffer = new OffHeapAppendMemory(64); + } + auxBuffer.putInt(globalId); + if (globalId > maxGlobalSymbolId) { + maxGlobalSymbolId = globalId; + } + valueCount++; + size++; + return; + } + storeGlobalSymbolIdsOnly = true; + } + dataBuffer.putInt(globalId); + + if (globalId > maxGlobalSymbolId) { + maxGlobalSymbolId = globalId; + } + + valueCount++; + size++; + } + + public void addUuid(long high, long low) { + // Store in wire order: lo first, hi second + dataBuffer.putLong(low); + dataBuffer.putLong(high); + valueCount++; + size++; + } + + @Override + public void close() { + if (dataBuffer != null) { + dataBuffer.close(); + dataBuffer = null; + } + if (auxBuffer != null) { + auxBuffer.close(); + auxBuffer = null; + } + if (stringOffsets != null) { + stringOffsets.close(); + stringOffsets = null; + } + if (stringData != null) { + stringData.close(); + stringData = null; + } + if (nullBufPtr != 0) { + Unsafe.free(nullBufPtr, (long) nullBufCapRows >>> 3, MemoryTag.NATIVE_ILP_RSS); + nullBufPtr = 0; + nullBufCapRows = 0; + } + } + + public void ensureNullBitmapCapacity(int minRows) { + if (nullBufPtr == 0 || nullBufCapRows >= minRows) { + return; + } + int newCapRows = Math.max(nullBufCapRows * 2, ((minRows + 63) >>> 6) << 6); + long newSizeBytes = (long) newCapRows >>> 3; + long oldSizeBytes = (long) nullBufCapRows >>> 3; + nullBufPtr = Unsafe.realloc(nullBufPtr, oldSizeBytes, newSizeBytes, MemoryTag.NATIVE_ILP_RSS); + Vect.memset(nullBufPtr + oldSizeBytes, newSizeBytes - oldSizeBytes, 0); + nullBufCapRows = newCapRows; + } + + public int getArrayDataOffset() { + return arrayDataOffset; + } + + public byte[] getArrayDims() { + return arrayDims; + } + + public int getArrayShapeOffset() { + return arrayShapeOffset; + } + + public int[] getArrayShapes() { + return arrayShapes; + } + + /** + * Returns the off-heap address of the auxiliary data buffer (global symbol IDs). + * Returns 0 if no auxiliary data exists. + */ + public long getAuxDataAddress() { + return auxBuffer != null ? auxBuffer.pageAddress() : 0; + } + + /** + * Returns the total bytes buffered in this column's storage. + */ + public long getBufferedBytes() { + long bytes = 0; + if (dataBuffer != null) { + bytes += dataBuffer.getAppendOffset(); + } + if (auxBuffer != null) { + bytes += auxBuffer.getAppendOffset(); + } + if (stringData != null) { + bytes += stringData.getAppendOffset(); + } + if (stringOffsets != null) { + bytes += stringOffsets.getAppendOffset(); + } + if (doubleArrayData != null) { + bytes += (long) arrayDataOffset * Double.BYTES; + } + if (longArrayData != null) { + bytes += (long) arrayDataOffset * Long.BYTES; + } + return bytes; + } + + /** + * Returns the off-heap address of the column data buffer. + */ + public long getDataAddress() { + return dataBuffer != null ? dataBuffer.pageAddress() : 0; + } + + public byte getDecimalScale() { + return decimalScale; + } + + public double[] getDoubleArrayData() { + return doubleArrayData; + } + + public int getGeoHashPrecision() { + return geohashPrecision; + } + + public int getIndex() { + return index; + } + + public long[] getLongArrayData() { + return longArrayData; + } + + public int getMaxGlobalSymbolId() { + return maxGlobalSymbolId; + } + + public String getName() { + return name; + } + + /** + * Returns the off-heap address of the null bitmap. + * Returns 0 for non-nullable columns. + */ + public long getNullBitmapAddress() { + return nullBufPtr; + } + + public int getSize() { + return size; + } + + public long getStringDataAddress() { + return stringData != null ? stringData.pageAddress() : 0; + } + + public long getStringDataSize() { + return stringData != null ? stringData.getAppendOffset() : 0; + } + + public long getStringOffsetsAddress() { + return stringOffsets != null ? stringOffsets.pageAddress() : 0; + } + + public String[] getSymbolDictionary() { + if (symbolList == null) { + return new String[0]; + } + String[] dict = new String[symbolList.size()]; + for (int i = 0; i < symbolList.size(); i++) { + dict[i] = symbolList.get(i); + } + return dict; + } + + public int getSymbolDictionarySize() { + if (storeGlobalSymbolIdsOnly) { + return 0; + } + return symbolList != null ? symbolList.size() : 0; + } + + public CharSequence getSymbolValue(int index) { + return symbolList != null ? symbolList.getQuick(index) : null; + } + + public byte getType() { + return type; + } + + public int getValueCount() { + return valueCount; + } + + public boolean hasSymbol(CharSequence value) { + return symbolDict != null && symbolDict.get(value) != CharSequenceIntHashMap.NO_ENTRY_VALUE; + } + + public boolean isNull(int index) { + if (nullBufPtr == 0 || index >= nullBufCapRows) { + return false; + } + long longAddr = nullBufPtr + ((long) (index >>> 6)) * 8; + int bitIndex = index & 63; + return (Unsafe.getUnsafe().getLong(longAddr) & (1L << bitIndex)) != 0; + } + + public void reset() { + size = 0; + valueCount = 0; + hasNulls = false; + if (dataBuffer != null) { + dataBuffer.truncate(); + } + if (auxBuffer != null) { + auxBuffer.truncate(); + } + if (stringOffsets != null) { + stringOffsets.truncate(); + stringOffsets.putInt(0); // re-seed initial 0 offset + } + if (stringData != null) { + stringData.truncate(); + } + if (nullBufPtr != 0) { + Vect.memset(nullBufPtr, (long) nullBufCapRows >>> 3, 0); + } + if (symbolDict != null) { + symbolDict.clear(); + symbolList.clear(); + } + storeGlobalSymbolIdsOnly = false; + maxGlobalSymbolId = -1; + arrayShapeOffset = 0; + arrayDataOffset = 0; + decimalScale = -1; + geohashPrecision = -1; + } + + public void retainTailRow( + int sizeBefore, + int valueCountBefore, + int arrayShapeOffsetBefore, + int arrayDataOffsetBefore + ) { + assert size == sizeBefore + 1; + + compactNullBitmap(sizeBefore); + + if (valueCount == valueCountBefore) { + clearValuePayload(); + size = 1; + valueCount = 0; + return; + } + + switch (type) { + case TYPE_STRING: + case TYPE_VARCHAR: + retainStringValue(valueCountBefore); + break; + case TYPE_SYMBOL: + retainSymbolValue(valueCountBefore); + break; + case TYPE_DOUBLE_ARRAY: + case TYPE_LONG_ARRAY: + retainArrayValue(valueCountBefore, arrayShapeOffsetBefore, arrayDataOffsetBefore); + break; + default: + retainFixedWidthValue(valueCountBefore); + break; + } + + size = 1; + valueCount = 1; + } + + public void truncateTo(int newSize) { + if (newSize >= size) { + return; + } + + int newValueCount = 0; + if (useNullBitmap && nullBufPtr != 0) { + for (int i = 0; i < newSize; i++) { + if (!isNull(i)) { + newValueCount++; + } + } + // Clear null bits for truncated rows + for (int i = newSize; i < Math.min(size, nullBufCapRows); i++) { + long longAddr = nullBufPtr + ((long) (i >>> 6)) * 8; + int bitIndex = i & 63; + long current = Unsafe.getUnsafe().getLong(longAddr); + Unsafe.getUnsafe().putLong(longAddr, current & ~(1L << bitIndex)); + } + hasNulls = false; + for (int i = 0; i < newSize && !hasNulls; i++) { + if (isNull(i)) { + hasNulls = true; + } + } + } else { + newValueCount = newSize; + } + + size = newSize; + valueCount = newValueCount; + + // Rewind off-heap data buffer + if (dataBuffer != null && elemSize > 0) { + dataBuffer.jumpTo((long) newValueCount * elemSize); + } + + // Rewind string buffers + if (stringOffsets != null) { + int dataOffset = Unsafe.getUnsafe().getInt(stringOffsets.pageAddress() + (long) newValueCount * 4); + stringData.jumpTo(dataOffset); + stringOffsets.jumpTo((long) (newValueCount + 1) * 4); + } + + // Rewind aux buffer (symbol global IDs) + if (auxBuffer != null) { + auxBuffer.jumpTo((long) newValueCount * 4); + } + + // Rewind array offsets by walking the retained values + if (arrayDims != null) { + int newShapeOffset = 0; + int newDataOffset = 0; + for (int i = 0; i < newValueCount; i++) { + int nDims = arrayDims[i]; + int elemCount = 1; + for (int d = 0; d < nDims; d++) { + elemCount *= arrayShapes[newShapeOffset++]; + } + newDataOffset += elemCount; + } + arrayShapeOffset = newShapeOffset; + arrayDataOffset = newDataOffset; + } + + // When all values are removed, reset type-specific metadata so the + // column behaves as freshly created (matches what reset() does). + if (newValueCount == 0) { + decimalScale = -1; + geohashPrecision = -1; + maxGlobalSymbolId = -1; + storeGlobalSymbolIdsOnly = false; + if (symbolDict != null) { + symbolDict.clear(); + symbolList.clear(); + } + } + } + + public boolean usesNullBitmap() { + return useNullBitmap; + } + + private static int checkedElementCount(long product) { + if (product > Integer.MAX_VALUE) { + throw new LineSenderException("array too large: total element count exceeds int range"); + } + return (int) product; + } + + private void allocateStorage(byte type) { + switch (type) { + case TYPE_BOOLEAN: + case TYPE_BYTE: + dataBuffer = new OffHeapAppendMemory(16); + break; + case TYPE_SHORT: + case TYPE_CHAR: + dataBuffer = new OffHeapAppendMemory(32); + break; + case TYPE_INT: + case TYPE_FLOAT: + dataBuffer = new OffHeapAppendMemory(64); + break; + case TYPE_GEOHASH: + case TYPE_LONG: + case TYPE_TIMESTAMP: + case TYPE_TIMESTAMP_NANOS: + case TYPE_DATE: + case TYPE_DECIMAL64: + case TYPE_DOUBLE: + dataBuffer = new OffHeapAppendMemory(128); + break; + case TYPE_UUID: + case TYPE_DECIMAL128: + dataBuffer = new OffHeapAppendMemory(256); + break; + case TYPE_LONG256: + case TYPE_DECIMAL256: + dataBuffer = new OffHeapAppendMemory(512); + break; + case TYPE_STRING: + case TYPE_VARCHAR: + stringOffsets = new OffHeapAppendMemory(64); + try { + stringOffsets.putInt(0); // seed initial 0 offset + stringData = new OffHeapAppendMemory(256); + } catch (Throwable t) { + stringOffsets.close(); + stringOffsets = null; + throw t; + } + break; + case TYPE_SYMBOL: + dataBuffer = new OffHeapAppendMemory(64); + symbolDict = new CharSequenceIntHashMap(); + symbolList = new ObjList<>(); + break; + case TYPE_DOUBLE_ARRAY: + case TYPE_LONG_ARRAY: + arrayDims = new byte[16]; + arrayCapture = new ArrayCapture(); + break; + } + } + + private void appendArrayPayload(long ptr, long len) { + if (len < 0) { + addNull(); + return; + } + if (len == 0) { + throw new LineSenderException("invalid array payload: empty payload"); + } + + int nDims = Unsafe.getUnsafe().getByte(ptr) & 0xFF; + if (nDims < 1 || nDims > ColumnType.ARRAY_NDIMS_LIMIT) { + throw new LineSenderException("invalid array payload: bad dimensionality " + nDims); + } + + long cursor = ptr + 1; + long headerBytes = 1L + (long) nDims * Integer.BYTES; + if (len < headerBytes) { + throw new LineSenderException("invalid array payload: truncated shape header"); + } + + int elemCount = 1; + for (int d = 0; d < nDims; d++) { + int dimLen = Unsafe.getUnsafe().getInt(cursor); + if (dimLen < 0) { + throw new LineSenderException("invalid array payload: negative dimension length"); + } + elemCount = checkedElementCount((long) elemCount * dimLen); + cursor += Integer.BYTES; + } + + long dataBytes = (long) elemCount * Double.BYTES; + if (len != headerBytes + dataBytes) { + throw new LineSenderException("invalid array payload: length mismatch"); + } + + ensureArrayCapacity(nDims, elemCount); + arrayDims[valueCount] = (byte) nDims; + + cursor = ptr + 1; + for (int d = 0; d < nDims; d++) { + arrayShapes[arrayShapeOffset++] = Unsafe.getUnsafe().getInt(cursor); + cursor += Integer.BYTES; + } + + if (dataBytes > 0) { + Unsafe.getUnsafe().copyMemory( + null, + cursor, + doubleArrayData, + DOUBLE_ARRAY_BASE_OFFSET + (long) arrayDataOffset * Double.BYTES, + dataBytes + ); + } + + arrayDataOffset += elemCount; + valueCount++; + size++; + } + + private void clearToEmptyFast() { + int sizeBefore = size; + clearValuePayload(); + if (nullBufPtr != 0 && sizeBefore > 0) { + int rowsToClear = Math.min(sizeBefore, nullBufCapRows); + long usedLongs = ((long) rowsToClear + 63) >>> 6; + Vect.memset(nullBufPtr, usedLongs * Long.BYTES, 0); + } + size = 0; + valueCount = 0; + hasNulls = false; + } + + private void clearValuePayload() { + if (dataBuffer != null && elemSize > 0) { + dataBuffer.jumpTo(0); + } + if (auxBuffer != null) { + auxBuffer.truncate(); + } + if (stringOffsets != null) { + stringOffsets.truncate(); + stringOffsets.putInt(0); + } + if (stringData != null) { + stringData.truncate(); + } + arrayShapeOffset = 0; + arrayDataOffset = 0; + resetEmptyMetadata(); + } + + private void compactNullBitmap(int sourceRow) { + if (nullBufPtr == 0) { + return; + } + + boolean retainedNull = isNull(sourceRow); + int rowsToClear = Math.min(size, nullBufCapRows); + long usedLongs = ((long) rowsToClear + 63) >>> 6; + Vect.memset(nullBufPtr, usedLongs * Long.BYTES, 0); + if (retainedNull) { + Unsafe.getUnsafe().putLong(nullBufPtr, 1L); + } + hasNulls = retainedNull; + } + + private void ensureArrayCapacity(int nDims, int dataElements) { + // Ensure per-row array dims capacity + if (valueCount >= arrayDims.length) { + arrayDims = Arrays.copyOf(arrayDims, arrayDims.length * 2); + } + + // Ensure shape array capacity + int requiredShapeCapacity = arrayShapeOffset + nDims; + if (arrayShapes == null) { + arrayShapes = new int[Math.max(64, requiredShapeCapacity)]; + } else if (requiredShapeCapacity > arrayShapes.length) { + arrayShapes = Arrays.copyOf(arrayShapes, Math.max(arrayShapes.length * 2, requiredShapeCapacity)); + } + + // Ensure data array capacity + int requiredDataCapacity = arrayDataOffset + dataElements; + if (type == TYPE_DOUBLE_ARRAY) { + if (doubleArrayData == null) { + doubleArrayData = new double[Math.max(256, requiredDataCapacity)]; + } else if (requiredDataCapacity > doubleArrayData.length) { + doubleArrayData = Arrays.copyOf(doubleArrayData, Math.max(doubleArrayData.length * 2, requiredDataCapacity)); + } + } else if (type == TYPE_LONG_ARRAY) { + if (longArrayData == null) { + longArrayData = new long[Math.max(256, requiredDataCapacity)]; + } else if (requiredDataCapacity > longArrayData.length) { + longArrayData = Arrays.copyOf(longArrayData, Math.max(longArrayData.length * 2, requiredDataCapacity)); + } + } + } + + private int getOrAddLocalSymbol(CharSequence value) { + int idx = symbolDict.get(value); + if (idx == CharSequenceIntHashMap.NO_ENTRY_VALUE) { + String symbol = Chars.toString(value); + idx = symbolList.size(); + symbolDict.put(symbol, idx); + symbolList.add(symbol); + } + return idx; + } + + private void markNull(int index) { + long longAddr = nullBufPtr + ((long) (index >>> 6)) * 8; + int bitIndex = index & 63; + long current = Unsafe.getUnsafe().getLong(longAddr); + Unsafe.getUnsafe().putLong(longAddr, current | (1L << bitIndex)); + hasNulls = true; + } + + private void resetEmptyMetadata() { + decimalScale = -1; + geohashPrecision = -1; + maxGlobalSymbolId = -1; + storeGlobalSymbolIdsOnly = false; + if (symbolDict != null) { + symbolDict.clear(); + symbolList.clear(); + } + } + + private void retainArrayValue(int valueIndex, int shapeOffsetBefore, int dataOffsetBefore) { + int nDims = arrayDims[valueIndex] & 0xFF; + arrayDims[0] = (byte) nDims; + + int shapeCount = arrayShapeOffset - shapeOffsetBefore; + if (shapeCount > 0 && shapeOffsetBefore > 0) { + System.arraycopy(arrayShapes, shapeOffsetBefore, arrayShapes, 0, shapeCount); + } + arrayShapeOffset = shapeCount; + + int dataCount = arrayDataOffset - dataOffsetBefore; + if (dataCount > 0 && dataOffsetBefore > 0) { + if (type == TYPE_LONG_ARRAY) { + System.arraycopy(longArrayData, dataOffsetBefore, longArrayData, 0, dataCount); + } else { + System.arraycopy(doubleArrayData, dataOffsetBefore, doubleArrayData, 0, dataCount); + } + } + arrayDataOffset = dataCount; + } + + private void retainFixedWidthValue(int valueIndex) { + if (dataBuffer == null || elemSize == 0) { + return; + } + + long srcOffset = (long) valueIndex * elemSize; + long dataAddress = dataBuffer.pageAddress(); + if (srcOffset > 0) { + Vect.memmove(dataAddress, dataAddress + srcOffset, elemSize); + } + dataBuffer.jumpTo(elemSize); + + if (auxBuffer != null) { + long auxAddress = auxBuffer.pageAddress(); + long auxOffset = (long) valueIndex * Integer.BYTES; + if (auxOffset > 0) { + Vect.memmove(auxAddress, auxAddress + auxOffset, Integer.BYTES); + } + auxBuffer.jumpTo(Integer.BYTES); + maxGlobalSymbolId = Unsafe.getUnsafe().getInt(auxAddress); + } + } + + private void retainStringValue(int valueIndex) { + long offsetsAddress = stringOffsets.pageAddress(); + int start = Unsafe.getUnsafe().getInt(offsetsAddress + (long) valueIndex * Integer.BYTES); + int end = Unsafe.getUnsafe().getInt(offsetsAddress + (long) (valueIndex + 1) * Integer.BYTES); + int len = end - start; + + if (len > 0 && start > 0) { + Vect.memmove(stringData.pageAddress(), stringData.pageAddress() + start, len); + } + + stringData.jumpTo(len); + stringOffsets.truncate(); + stringOffsets.putInt(0); + stringOffsets.putInt(len); + } + + private void retainSymbolValue(int valueIndex) { + retainFixedWidthValue(valueIndex); + + if (storeGlobalSymbolIdsOnly) { + maxGlobalSymbolId = Unsafe.getUnsafe().getInt(dataBuffer.pageAddress()); + return; + } + + int localIndex = Unsafe.getUnsafe().getInt(dataBuffer.pageAddress()); + String symbol = symbolList.get(localIndex); + + symbolDict.clear(); + symbolList.clear(); + symbolList.add(symbol); + symbolDict.put(symbol, 0); + Unsafe.getUnsafe().putInt(dataBuffer.pageAddress(), 0); + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java new file mode 100644 index 0000000..3a86bf6 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java @@ -0,0 +1,147 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.websocket; + +/** + * WebSocket close status codes as defined in RFC 6455. + */ +public final class WebSocketCloseCode { + /** + * Abnormal closure (1006). + * Reserved value. MUST NOT be sent in a Close frame. + * Used to indicate that a connection was closed abnormally. + */ + public static final int ABNORMAL_CLOSURE = 1006; + /** + * Going away (1001). + * The endpoint is going away, e.g., server shutting down or browser navigating away. + */ + public static final int GOING_AWAY = 1001; + /** + * Internal server error (1011). + * The server encountered an unexpected condition that prevented it from fulfilling the request. + */ + public static final int INTERNAL_ERROR = 1011; + /** + * Invalid frame payload data (1007). + * The endpoint received a message with invalid payload data. + */ + public static final int INVALID_PAYLOAD_DATA = 1007; + /** + * Mandatory extension (1010). + * The client expected the server to negotiate one or more extensions. + */ + public static final int MANDATORY_EXTENSION = 1010; + /** + * Message too big (1009). + * The endpoint received a message that is too big to process. + */ + public static final int MESSAGE_TOO_BIG = 1009; + /** + * Normal closure (1000). + * The connection successfully completed whatever purpose for which it was created. + */ + public static final int NORMAL_CLOSURE = 1000; + /** + * No status received (1005). + * Reserved value. MUST NOT be sent in a Close frame. + */ + public static final int NO_STATUS_RECEIVED = 1005; + /** + * Policy violation (1008). + * The endpoint received a message that violates its policy. + */ + public static final int POLICY_VIOLATION = 1008; + /** + * Protocol error (1002). + * The endpoint is terminating the connection due to a protocol error. + */ + public static final int PROTOCOL_ERROR = 1002; + /** + * Reserved (1004). + * Reserved for future use. + */ + public static final int RESERVED = 1004; + /** + * TLS handshake (1015). + * Reserved value. MUST NOT be sent in a Close frame. + * Used to indicate that the connection was closed due to TLS handshake failure. + */ + public static final int TLS_HANDSHAKE = 1015; + /** + * Unsupported data (1003). + * The endpoint received a type of data it cannot accept. + */ + public static final int UNSUPPORTED_DATA = 1003; + + private WebSocketCloseCode() { + // Constants class + } + + /** + * Returns a human-readable description of the close code. + * + * @param code the close code + * @return the description + */ + public static String describe(int code) { + switch (code) { + case NORMAL_CLOSURE: + return "Normal Closure"; + case GOING_AWAY: + return "Going Away"; + case PROTOCOL_ERROR: + return "Protocol Error"; + case UNSUPPORTED_DATA: + return "Unsupported Data"; + case RESERVED: + return "Reserved"; + case NO_STATUS_RECEIVED: + return "No Status Received"; + case ABNORMAL_CLOSURE: + return "Abnormal Closure"; + case INVALID_PAYLOAD_DATA: + return "Invalid Payload Data"; + case POLICY_VIOLATION: + return "Policy Violation"; + case MESSAGE_TOO_BIG: + return "Message Too Big"; + case MANDATORY_EXTENSION: + return "Mandatory Extension"; + case INTERNAL_ERROR: + return "Internal Error"; + case TLS_HANDSHAKE: + return "TLS Handshake"; + default: + if (code >= 3000 && code < 4000) { + return "Library/Framework Code (" + code + ")"; + } else if (code >= 4000 && code < 5000) { + return "Application Code (" + code + ")"; + } + return "Unknown (" + code + ")"; + } + } + +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java new file mode 100644 index 0000000..ca6cab9 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java @@ -0,0 +1,224 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.websocket; + +import io.questdb.client.std.Unsafe; + +/** + * Zero-allocation WebSocket frame parser. + * Parses WebSocket frames according to RFC 6455. + * + *

    The parser operates on raw memory buffers and maintains minimal state. + * It can parse frames incrementally when data arrives in chunks. + * + *

    Thread safety: This class is NOT thread-safe. Each connection should + * have its own parser instance. + */ +public class WebSocketFrameParser { + /** + * Frame completely parsed. + */ + public static final int STATE_COMPLETE = 3; + /** + * Error state - frame is invalid. + */ + public static final int STATE_ERROR = 4; + /** + * Initial state, waiting for frame header. + */ + public static final int STATE_HEADER = 0; + /** + * Need more data to complete parsing. + */ + public static final int STATE_NEED_MORE = 1; + /** + * Header parsed, need payload data. + */ + public static final int STATE_NEED_PAYLOAD = 2; + // Frame header bits + private static final int FIN_BIT = 0x80; + private static final int LENGTH_MASK = 0x7F; + // Control frame max payload size (RFC 6455) + private static final int MAX_CONTROL_FRAME_PAYLOAD = 125; + private static final int OPCODE_MASK = 0x0F; + private static final int RSV_BITS = 0x70; + private int errorCode; + // Parsed frame data + private boolean fin; + private int headerSize; + private int opcode; + private long payloadLength; + // Parser state + private int state = STATE_HEADER; + + public int getErrorCode() { + return errorCode; + } + + public int getHeaderSize() { + return headerSize; + } + + public int getOpcode() { + return opcode; + } + + public long getPayloadLength() { + return payloadLength; + } + + public int getState() { + return state; + } + + public boolean isFin() { + return fin; + } + + /** + * Parses a WebSocket frame from the given buffer. + * + * @param buf the start of the buffer + * @param limit the end of the buffer (exclusive) + * @return the number of bytes consumed, or 0 if more data is needed + */ + public int parse(long buf, long limit) { + long available = limit - buf; + + if (available < 2) { + state = STATE_NEED_MORE; + return 0; + } + + // Parse first two bytes + int byte0 = Unsafe.getUnsafe().getByte(buf) & 0xFF; + int byte1 = Unsafe.getUnsafe().getByte(buf + 1) & 0xFF; + + // Check reserved bits (must be 0 unless extension negotiated) + if ((byte0 & RSV_BITS) != 0) { + state = STATE_ERROR; + errorCode = WebSocketCloseCode.PROTOCOL_ERROR; + return 0; + } + + fin = (byte0 & FIN_BIT) != 0; + opcode = byte0 & OPCODE_MASK; + + // Validate opcode + if (!WebSocketOpcode.isValid(opcode)) { + state = STATE_ERROR; + errorCode = WebSocketCloseCode.PROTOCOL_ERROR; + return 0; + } + + // Control frames must not be fragmented + if (WebSocketOpcode.isControlFrame(opcode) && !fin) { + state = STATE_ERROR; + errorCode = WebSocketCloseCode.PROTOCOL_ERROR; + return 0; + } + + int lengthField = byte1 & LENGTH_MASK; + + // Server frames MUST NOT be masked (RFC 6455 section 5.1) + if ((byte1 & 0x80) != 0) { + state = STATE_ERROR; + errorCode = WebSocketCloseCode.PROTOCOL_ERROR; + return 0; + } + + // Calculate header size and payload length + int offset = 2; + + if (lengthField <= 125) { + payloadLength = lengthField; + } else if (lengthField == 126) { + // 16-bit extended length + if (available < 4) { + state = STATE_NEED_MORE; + return 0; + } + int high = Unsafe.getUnsafe().getByte(buf + 2) & 0xFF; + int low = Unsafe.getUnsafe().getByte(buf + 3) & 0xFF; + payloadLength = (high << 8) | low; + offset = 4; + } else { + // 64-bit extended length + if (available < 10) { + state = STATE_NEED_MORE; + return 0; + } + payloadLength = Long.reverseBytes(Unsafe.getUnsafe().getLong(buf + 2)); + + // MSB must be 0 (no negative lengths) + if (payloadLength < 0) { + state = STATE_ERROR; + errorCode = WebSocketCloseCode.PROTOCOL_ERROR; + return 0; + } + + offset = 10; + } + + // Control frames must not have payload > 125 bytes + if (WebSocketOpcode.isControlFrame(opcode) && payloadLength > MAX_CONTROL_FRAME_PAYLOAD) { + state = STATE_ERROR; + errorCode = WebSocketCloseCode.PROTOCOL_ERROR; + return 0; + } + + // Close frame with 1 byte payload is invalid (must be 0 or >= 2) + if (opcode == WebSocketOpcode.CLOSE && payloadLength == 1) { + state = STATE_ERROR; + errorCode = WebSocketCloseCode.PROTOCOL_ERROR; + return 0; + } + + headerSize = offset; + + // Check if we have the complete payload + long totalFrameSize = headerSize + payloadLength; + if (available < totalFrameSize) { + state = STATE_NEED_PAYLOAD; + return headerSize; + } + + state = STATE_COMPLETE; + return (int) totalFrameSize; + } + + /** + * Resets the parser state for parsing a new frame. + */ + public void reset() { + state = STATE_HEADER; + fin = false; + opcode = 0; + payloadLength = 0; + headerSize = 0; + errorCode = 0; + } + +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java new file mode 100644 index 0000000..07ba22b --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java @@ -0,0 +1,161 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.websocket; + +import io.questdb.client.std.Unsafe; + +import java.nio.ByteOrder; + +/** + * Zero-allocation WebSocket frame writer. + * Writes WebSocket frames according to RFC 6455. + * + *

    All methods are static utilities that write directly to memory buffers. + * + *

    Thread safety: This class is thread-safe as it contains no mutable state. + */ +public final class WebSocketFrameWriter { + // Frame header bits + private static final int FIN_BIT = 0x80; + private static final boolean IS_BIG_ENDIAN = ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN; + private static final int MASK_BIT = 0x80; + + private WebSocketFrameWriter() { + // Static utility class + } + + /** + * Calculates the header size for a given payload length and masking. + * + * @param payloadLength the payload length + * @param masked true if the payload will be masked + * @return the header size in bytes + */ + public static int headerSize(long payloadLength, boolean masked) { + int size; + if (payloadLength <= 125) { + size = 2; + } else if (payloadLength <= 65535) { + size = 4; + } else { + size = 10; + } + return masked ? size + 4 : size; + } + + /** + * Masks payload data in place using XOR with the given mask key. + * + * @param buf the payload buffer + * @param len the payload length + * @param maskKey the 4-byte mask key + */ + public static void maskPayload(long buf, long len, int maskKey) { + // maskKey is in big-endian convention: MSB = wire byte 0 = mask byte for position 0. + // For bulk XOR via getInt/getLong (native byte order), convert to native order + // so that memory position 0 XORs with mask byte 0, position 1 with mask byte 1, etc. + int nativeMask = IS_BIG_ENDIAN ? maskKey : Integer.reverseBytes(maskKey); + long longMask = ((long) nativeMask << 32) | (nativeMask & 0xFFFFFFFFL); + + long i = 0; + + // Process 8-byte chunks + while (i + 8 <= len) { + long value = Unsafe.getUnsafe().getLong(buf + i); + Unsafe.getUnsafe().putLong(buf + i, value ^ longMask); + i += 8; + } + + // Process 4-byte chunk if remaining + if (i + 4 <= len) { + int value = Unsafe.getUnsafe().getInt(buf + i); + Unsafe.getUnsafe().putInt(buf + i, value ^ nativeMask); + i += 4; + } + + // Process remaining bytes - extract mask byte in big-endian order + while (i < len) { + byte b = Unsafe.getUnsafe().getByte(buf + i); + int maskByte = (maskKey >>> ((3 - ((int) i & 3)) << 3)) & 0xFF; + Unsafe.getUnsafe().putByte(buf + i, (byte) (b ^ maskByte)); + i++; + } + } + + /** + * Writes a WebSocket frame header to the buffer. + * + * @param buf the buffer to write to + * @param fin true if this is the final frame + * @param opcode the frame opcode + * @param payloadLength the payload length + * @param masked true if the payload should be masked + * @return the number of bytes written (header size) + */ + public static int writeHeader(long buf, boolean fin, int opcode, long payloadLength, boolean masked) { + int offset = 0; + + // First byte: FIN + opcode + int byte0 = (fin ? FIN_BIT : 0) | (opcode & 0x0F); + Unsafe.getUnsafe().putByte(buf + offset++, (byte) byte0); + + // Second byte: MASK + payload length + int maskBit = masked ? MASK_BIT : 0; + + if (payloadLength <= 125) { + Unsafe.getUnsafe().putByte(buf + offset++, (byte) (maskBit | payloadLength)); + } else if (payloadLength <= 65535) { + Unsafe.getUnsafe().putByte(buf + offset++, (byte) (maskBit | 126)); + Unsafe.getUnsafe().putByte(buf + offset++, (byte) ((payloadLength >> 8) & 0xFF)); + Unsafe.getUnsafe().putByte(buf + offset++, (byte) (payloadLength & 0xFF)); + } else { + Unsafe.getUnsafe().putByte(buf + offset++, (byte) (maskBit | 127)); + Unsafe.getUnsafe().putLong(buf + offset, Long.reverseBytes(payloadLength)); + offset += 8; + } + + return offset; + } + + /** + * Writes a WebSocket frame header with optional mask key. + * + * @param buf the buffer to write to + * @param fin true if this is the final frame + * @param opcode the frame opcode + * @param payloadLength the payload length + * @param maskKey the mask key (only used if masked is true) + * @return the number of bytes written (header size including mask key) + */ + public static int writeHeader(long buf, boolean fin, int opcode, long payloadLength, int maskKey) { + int offset = writeHeader(buf, fin, opcode, payloadLength, true); + // Write mask key in network byte order (big-endian) per RFC 6455 + Unsafe.getUnsafe().putByte(buf + offset, (byte) (maskKey >>> 24)); + Unsafe.getUnsafe().putByte(buf + offset + 1, (byte) (maskKey >>> 16)); + Unsafe.getUnsafe().putByte(buf + offset + 2, (byte) (maskKey >>> 8)); + Unsafe.getUnsafe().putByte(buf + offset + 3, (byte) maskKey); + return offset + 4; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java new file mode 100644 index 0000000..74bb7bf --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java @@ -0,0 +1,107 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.websocket; + +/** + * WebSocket frame opcodes as defined in RFC 6455. + */ +public final class WebSocketOpcode { + /** + * Binary frame (0x2). + * Payload is arbitrary binary data. + */ + public static final int BINARY = 0x02; + /** + * Connection close frame (0x8). + * Indicates that the endpoint wants to close the connection. + */ + public static final int CLOSE = 0x08; + /** + * Continuation frame (0x0). + * Used for fragmented messages after the initial frame. + */ + public static final int CONTINUATION = 0x00; + + // Reserved non-control frames: 0x3-0x7 + /** + * Ping frame (0x9). + * Used for keep-alive and connection health checks. + */ + public static final int PING = 0x09; + /** + * Pong frame (0xA). + * Response to a ping frame. + */ + public static final int PONG = 0x0A; + /** + * Text frame (0x1). + * Payload is UTF-8 encoded text. + */ + public static final int TEXT = 0x01; + + // Reserved control frames: 0xB-0xF + + private WebSocketOpcode() { + // Constants class + } + + /** + * Checks if the opcode is a control frame. + * Control frames are CLOSE (0x8), PING (0x9), and PONG (0xA). + * + * @param opcode the opcode to check + * @return true if the opcode is a control frame + */ + public static boolean isControlFrame(int opcode) { + return (opcode & 0x08) != 0; + } + + /** + * Checks if the opcode is a data frame. + * Data frames are CONTINUATION (0x0), TEXT (0x1), and BINARY (0x2). + * + * @param opcode the opcode to check + * @return true if the opcode is a data frame + */ + public static boolean isDataFrame(int opcode) { + return opcode <= 0x02; + } + + /** + * Checks if the opcode is valid according to RFC 6455. + * + * @param opcode the opcode to check + * @return true if the opcode is valid + */ + public static boolean isValid(int opcode) { + return opcode == CONTINUATION + || opcode == TEXT + || opcode == BINARY + || opcode == CLOSE + || opcode == PING + || opcode == PONG; + } + +} diff --git a/core/src/main/java/io/questdb/client/network/IOOperation.java b/core/src/main/java/io/questdb/client/network/IOOperation.java index e32b1fb..eb8afda 100644 --- a/core/src/main/java/io/questdb/client/network/IOOperation.java +++ b/core/src/main/java/io/questdb/client/network/IOOperation.java @@ -25,7 +25,6 @@ package io.questdb.client.network; public final class IOOperation { - public static final int HEARTBEAT = 8; public static final int READ = 1; public static final int WRITE = 4; diff --git a/core/src/main/java/io/questdb/client/network/Net.java b/core/src/main/java/io/questdb/client/network/Net.java index a4688e1..9e65461 100644 --- a/core/src/main/java/io/questdb/client/network/Net.java +++ b/core/src/main/java/io/questdb/client/network/Net.java @@ -80,9 +80,9 @@ public static void configureKeepAlive(int fd) { public static native int configureNonBlocking(int fd); - public native static int connect(int fd, long sockaddr); + public static native int connect(int fd, long sockaddr); - public native static int connectAddrInfo(int fd, long lpAddrInfo); + public static native int connectAddrInfo(int fd, long lpAddrInfo); public static void freeAddrInfo(long pAddrInfo) { if (pAddrInfo != 0) { @@ -106,63 +106,61 @@ public static long getAddrInfo(CharSequence host, int port) { } } - private static long getAddrInfo(DirectUtf8Sequence host, int port) { - return getAddrInfo(host.ptr(), port); - } - - private static long getAddrInfo(long lpszHost, int port) { - long addrInfo = getAddrInfo0(lpszHost, port); - if (addrInfo != -1) { - ADDR_INFO_COUNTER.incrementAndGet(); - } - return addrInfo; - } - - public native static int getSndBuf(int fd); + public static native int getSndBuf(int fd); public static void init() { // no-op } - public native static boolean join(int fd, int bindIPv4Address, int groupIPv4Address); + public static native boolean join(int fd, int bindIPv4Address, int groupIPv4Address); public static native int peek(int fd, long ptr, int len); public static native int recv(int fd, long ptr, int len); - public static int send(long fd, long ptr, int len) { - return send(fd, ptr, len); - } - public static native int send(int fd, long ptr, int len); - public native static int sendTo(int fd, long ptr, int len, long sockaddr); + public static native int sendTo(int fd, long ptr, int len, long sockaddr); + + public static native int sendToScatter(int fd, long segmentsPtr, int segmentCount, long sockaddr); public static native int setKeepAlive0(int fd, int seconds); - public native static int setMulticastInterface(int fd, int ipv4address); + public static native int setMulticastInterface(int fd, int ipv4address); - public native static int setMulticastTtl(int fd, int ttl); + public static native int setMulticastTtl(int fd, int ttl); - public native static int setSndBuf(int fd, int size); + public static native int setSndBuf(int fd, int size); - public native static int setTcpNoDelay(int fd, boolean noDelay); + public static native int setTcpNoDelay(int fd, boolean noDelay); public static long sockaddr(int ipv4address, int port) { SOCK_ADDR_COUNTER.incrementAndGet(); return sockaddr0(ipv4address, port); } - public native static long sockaddr0(int ipv4address, int port); + public static native long sockaddr0(int ipv4address, int port); - public native static int socketTcp(boolean blocking); + public static native int socketTcp(boolean blocking); - public native static int socketUdp(); + public static native int socketUdp(); private static native void freeAddrInfo0(long pAddrInfo); private static native void freeSockAddr0(long sockaddr); + private static long getAddrInfo(DirectUtf8Sequence host, int port) { + return getAddrInfo(host.ptr(), port); + } + + private static long getAddrInfo(long lpszHost, int port) { + long addrInfo = getAddrInfo0(lpszHost, port); + if (addrInfo != -1) { + ADDR_INFO_COUNTER.incrementAndGet(); + } + return addrInfo; + } + private static native long getAddrInfo0(long lpszHost, int port); private static native int getEwouldblock(); @@ -186,4 +184,4 @@ public static long sockaddr(int ipv4address, int port) { MMSGHDR_BUFFER_LENGTH_OFFSET = -1L; } } -} \ No newline at end of file +} diff --git a/core/src/main/java/io/questdb/client/network/NetworkFacade.java b/core/src/main/java/io/questdb/client/network/NetworkFacade.java index 0551dc4..b2e97da 100644 --- a/core/src/main/java/io/questdb/client/network/NetworkFacade.java +++ b/core/src/main/java/io/questdb/client/network/NetworkFacade.java @@ -55,6 +55,8 @@ public interface NetworkFacade { int sendToRaw(int fd, long lo, int len, long socketAddress); + int sendToRawScatter(int fd, long segmentsPtr, int segmentCount, long socketAddress); + int setMulticastInterface(int fd, int ipv4Address); int setMulticastTtl(int fd, int ttl); @@ -78,4 +80,4 @@ public interface NetworkFacade { * @return true if a disconnect happened, false otherwise */ boolean testConnection(int fd, long buffer, int bufferSize); -} \ No newline at end of file +} diff --git a/core/src/main/java/io/questdb/client/network/NetworkFacadeImpl.java b/core/src/main/java/io/questdb/client/network/NetworkFacadeImpl.java index c50549c..11195fc 100644 --- a/core/src/main/java/io/questdb/client/network/NetworkFacadeImpl.java +++ b/core/src/main/java/io/questdb/client/network/NetworkFacadeImpl.java @@ -102,6 +102,11 @@ public int sendToRaw(int fd, long ptr, int len, long socketAddress) { return Net.sendTo(fd, ptr, len, socketAddress); } + @Override + public int sendToRawScatter(int fd, long segmentsPtr, int segmentCount, long socketAddress) { + return Net.sendToScatter(fd, segmentsPtr, segmentCount, socketAddress); + } + @Override public int setMulticastInterface(int fd, int ipv4Address) { return Net.setMulticastInterface(fd, ipv4Address); @@ -149,4 +154,4 @@ public boolean testConnection(int fd, long buffer, int bufferSize) { final int nRead = Net.peek(fd, buffer, bufferSize); return nRead < 0; } -} \ No newline at end of file +} diff --git a/core/src/main/java/io/questdb/client/std/AbstractLowerCaseAsciiCharSequenceHashSet.java b/core/src/main/java/io/questdb/client/std/AbstractLowerCaseAsciiCharSequenceHashSet.java index 91bbc5f..4424999 100644 --- a/core/src/main/java/io/questdb/client/std/AbstractLowerCaseAsciiCharSequenceHashSet.java +++ b/core/src/main/java/io/questdb/client/std/AbstractLowerCaseAsciiCharSequenceHashSet.java @@ -54,13 +54,13 @@ public void clear() { } public int keyIndex(CharSequence key) { - int index = Chars.lowerCaseAsciiHashCode(key) & mask; + int index = Chars.lowerCaseHashCode(key) & mask; if (keys[index] == noEntryKey) { return index; } - if (Chars.equalsLowerCaseAscii(key, keys[index])) { + if (Chars.equalsIgnoreCase(key, keys[index])) { return -index - 1; } @@ -77,7 +77,7 @@ private int probe(CharSequence key, int index) { if (keys[index] == noEntryKey) { return index; } - if (Chars.equalsLowerCaseAscii(key, keys[index])) { + if (Chars.equalsIgnoreCase(key, keys[index])) { return -index - 1; } } while (true); diff --git a/core/src/main/java/io/questdb/client/std/AbstractLowerCaseCharSequenceHashSet.java b/core/src/main/java/io/questdb/client/std/AbstractLowerCaseCharSequenceHashSet.java deleted file mode 100644 index e2db14c..0000000 --- a/core/src/main/java/io/questdb/client/std/AbstractLowerCaseCharSequenceHashSet.java +++ /dev/null @@ -1,89 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.std; - -import java.util.Arrays; - -public abstract class AbstractLowerCaseCharSequenceHashSet implements Mutable { - protected static final int MIN_INITIAL_CAPACITY = 16; - protected static final CharSequence noEntryKey = null; - protected final double loadFactor; - protected int capacity; - protected int free; - protected CharSequence[] keys; - protected int mask; - - public AbstractLowerCaseCharSequenceHashSet(int initialCapacity, double loadFactor) { - if (loadFactor <= 0d || loadFactor >= 1d) { - throw new IllegalArgumentException("0 < loadFactor < 1"); - } - - free = this.capacity = Math.max(initialCapacity, MIN_INITIAL_CAPACITY); - this.loadFactor = loadFactor; - keys = new CharSequence[Numbers.ceilPow2((int) (this.capacity / loadFactor))]; - mask = keys.length - 1; - } - - @Override - public void clear() { - Arrays.fill(keys, noEntryKey); - free = capacity; - } - - public boolean contains(CharSequence key) { - return keyIndex(key) < 0; - } - - public int keyIndex(CharSequence key) { - int index = Chars.lowerCaseHashCode(key) & mask; - - if (keys[index] == noEntryKey) { - return index; - } - - if (Chars.equalsIgnoreCase(key, keys[index])) { - return -index - 1; - } - - return probe(key, index); - } - - public int size() { - return capacity - free; - } - - private int probe(CharSequence key, int index) { - do { - index = (index + 1) & mask; - if (keys[index] == noEntryKey) { - return index; - } - if (Chars.equalsIgnoreCase(key, keys[index])) { - return -index - 1; - } - } while (true); - } - -} \ No newline at end of file diff --git a/core/src/main/java/io/questdb/client/std/Base64Helper.java b/core/src/main/java/io/questdb/client/std/Base64Helper.java deleted file mode 100644 index 5f29bc1..0000000 --- a/core/src/main/java/io/questdb/client/std/Base64Helper.java +++ /dev/null @@ -1,116 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -// Written by Gil Tene of Azul Systems, and released to the public domain, -// as explained at http://creativecommons.org/publicdomain/zero/1.0/ -// -// @author Gil Tene - -package io.questdb.client.std; - -import java.lang.reflect.Method; - -/** - * Base64Helper exists to bridge inconsistencies in Java SE support of Base64 encoding and decoding. - * Earlier Java SE platforms (up to and including Java SE 8) supported base64 encode/decode via the - * javax.xml.bind.DatatypeConverter class, which was deprecated and eventually removed in Java SE 9. - * Later Java SE platforms (Java SE 8 and later) support base64 encode/decode via the - * java.util.Base64 class (first introduced in Java SE 8, and not available on e.g. Java SE 6 or 7). - *

    - * This makes it "hard" to write a single piece of source code that deals with base64 encodings and - * will compile and run on e.g. Java SE 7 AND Java SE 9. And such common source is a common need for - * libraries. This class is intended to encapsulate this "hard"-ness and hide the ugly pretzle-twising - * needed under the covers. - *

    - * Base64Helper provides a common API that works across Java SE 6..9 (and beyond hopefully), and - * uses late binding (Reflection) internally to avoid javac-compile-time dependencies on a specific - * Java SE version (e.g. beyond 7 or before 9). - */ -public class Base64Helper { - - private static Method decodeMethod; - // encoderObj and decoderObj are used in non-static method forms, and - // irrelevant for static method forms: - private static Object decoderObj; - private static Method encodeMethod; - private static Object encoderObj; - - /** - * Converts a Base64 encoded String to a byte array - * - * @param base64input A base64-encoded input String - * @return a byte array containing the binary representation equivalent of the Base64 encoded input - */ - public static byte[] parseBase64Binary(String base64input) { - try { - return (byte[]) decodeMethod.invoke(decoderObj, base64input); - } catch (Throwable e) { - throw new UnsupportedOperationException("Failed to use platform's base64 decode method"); - } - } - - /** - * Converts an array of bytes into a Base64 string. - * - * @param binaryArray A binary encoded input array - * @return a String containing the Base64 encoded equivalent of the binary input - */ - static String printBase64Binary(byte[] binaryArray) { - try { - return (String) encodeMethod.invoke(encoderObj, binaryArray); - } catch (Throwable e) { - throw new UnsupportedOperationException("Failed to use platform's base64 encode method"); - } - } - - static { - try { - Class javaUtilBase64Class = Class.forName("java.util.Base64"); - - Method getDecoderMethod = javaUtilBase64Class.getMethod("getDecoder"); - decoderObj = getDecoderMethod.invoke(null); - decodeMethod = decoderObj.getClass().getMethod("decode", String.class); - - Method getEncoderMethod = javaUtilBase64Class.getMethod("getEncoder"); - encoderObj = getEncoderMethod.invoke(null); - encodeMethod = encoderObj.getClass().getMethod("encodeToString", byte[].class); - } catch (Throwable e) { - decodeMethod = null; - encodeMethod = null; - } - - if (encodeMethod == null) { - decoderObj = null; - encoderObj = null; - try { - Class javaxXmlBindDatatypeConverterClass = Class.forName("javax.xml.bind.DatatypeConverter"); - decodeMethod = javaxXmlBindDatatypeConverterClass.getMethod("parseBase64Binary", String.class); - encodeMethod = javaxXmlBindDatatypeConverterClass.getMethod("printBase64Binary", byte[].class); - } catch (Throwable e) { - decodeMethod = null; - encodeMethod = null; - } - } - } -} diff --git a/core/src/main/java/io/questdb/client/std/BiIntFunction.java b/core/src/main/java/io/questdb/client/std/BiIntFunction.java deleted file mode 100644 index 6300066..0000000 --- a/core/src/main/java/io/questdb/client/std/BiIntFunction.java +++ /dev/null @@ -1,30 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.std; - -@FunctionalInterface -public interface BiIntFunction { - R apply(int val1, U val2); -} diff --git a/core/src/main/java/io/questdb/client/std/BufferWindowCharSequence.java b/core/src/main/java/io/questdb/client/std/BufferWindowCharSequence.java deleted file mode 100644 index 097b65f..0000000 --- a/core/src/main/java/io/questdb/client/std/BufferWindowCharSequence.java +++ /dev/null @@ -1,29 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.std; - -public interface BufferWindowCharSequence extends CharSequence { - -} \ No newline at end of file diff --git a/core/src/main/java/io/questdb/client/std/CharSequenceIntHashMap.java b/core/src/main/java/io/questdb/client/std/CharSequenceIntHashMap.java new file mode 100644 index 0000000..29b4e39 --- /dev/null +++ b/core/src/main/java/io/questdb/client/std/CharSequenceIntHashMap.java @@ -0,0 +1,211 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.std; + +import org.jetbrains.annotations.NotNull; + +import java.util.Arrays; + + +public class CharSequenceIntHashMap extends AbstractCharSequenceHashSet { + public static final int NO_ENTRY_VALUE = -1; + private final ObjList list; + private final int noEntryValue; + private int[] values; + + public CharSequenceIntHashMap() { + this(8); + } + + public CharSequenceIntHashMap(int initialCapacity) { + this(initialCapacity, 0.4, NO_ENTRY_VALUE); + } + + public CharSequenceIntHashMap(int initialCapacity, double loadFactor, int noEntryValue) { + super(initialCapacity, loadFactor); + this.noEntryValue = noEntryValue; + this.list = new ObjList<>(capacity); + values = new int[keys.length]; + clear(); + } + + @Override + public final void clear() { + super.clear(); + list.clear(); + Arrays.fill(values, noEntryValue); + } + + public int get(@NotNull CharSequence key) { + return valueAt(keyIndex(key)); + } + + public void inc(@NotNull CharSequence key) { + int index = keyIndex(key); + if (index < 0) { + values[-index - 1]++; + } else { + String keyString = Chars.toString(key); + putAt0(index, keyString, 1); + list.add(keyString); + } + } + + public ObjList keys() { + return list; + } + + public boolean put(@NotNull CharSequence key, int value) { + return putAt(keyIndex(key), key, value); + } + + public void putAll(@NotNull CharSequenceIntHashMap other) { + CharSequence[] otherKeys = other.keys; + int[] otherValues = other.values; + for (int i = 0, n = otherKeys.length; i < n; i++) { + if (otherKeys[i] != noEntryKey) { + put(otherKeys[i], otherValues[i]); + } + } + } + + public boolean putAt(int index, @NotNull CharSequence key, int value) { + if (index < 0) { + values[-index - 1] = value; + return false; + } + final String keyString = Chars.toString(key); + putAt0(index, keyString, value); + list.add(keyString); + return true; + } + + public void putIfAbsent(@NotNull CharSequence key, int value) { + int index = keyIndex(key); + if (index > -1) { + String keyString = Chars.toString(key); + putAt0(index, keyString, value); + list.add(keyString); + } + } + + public void removeAt(int index) { + if (index < 0) { + int from = -index - 1; + CharSequence key = keys[from]; + erase(from); + free++; + + // after we have freed up a slot + // consider non-empty keys directly below + // they may have been a direct hit but because + // directly hit slot wasn't empty these keys would + // have moved. + // + // After slot is freed these keys require re-hash + from = (from + 1) & mask; + for ( + CharSequence k = keys[from]; + k != noEntryKey; + from = (from + 1) & mask, k = keys[from] + ) { + int idealHit = Hash.spread(Chars.hashCode(k)) & mask; + if (idealHit != from) { + int to; + if (keys[idealHit] != noEntryKey) { + to = probe0(k, idealHit); + } else { + to = idealHit; + } + + if (to > -1) { + move(from, to); + } + } + } + + list.remove(key); + } + } + + public int valueAt(int index) { + int index1 = -index - 1; + return index < 0 ? values[index1] : noEntryValue; + } + + public int valueQuick(int index) { + return get(list.getQuick(index)); + } + + private void erase(int index) { + keys[index] = noEntryKey; + values[index] = noEntryValue; + } + + private void move(int from, int to) { + keys[to] = keys[from]; + values[to] = values[from]; + erase(from); + } + + private int probe0(CharSequence key, int index) { + do { + index = (index + 1) & mask; + if (keys[index] == noEntryKey) { + return index; + } + if (Chars.equals(key, keys[index])) { + return -index - 1; + } + } while (true); + } + + private void putAt0(int index, CharSequence key, int value) { + keys[index] = key; + values[index] = value; + if (--free == 0) { + rehash(); + } + } + + private void rehash() { + int[] oldValues = values; + CharSequence[] oldKeys = keys; + int size = capacity - free; + capacity = capacity * 2; + free = capacity - size; + mask = Numbers.ceilPow2((int) (capacity / loadFactor)) - 1; + this.keys = new CharSequence[mask + 1]; + this.values = new int[mask + 1]; + for (int i = oldKeys.length - 1; i > -1; i--) { + CharSequence key = oldKeys[i]; + if (key != null) { + final int index = keyIndex(key); + keys[index] = key; + values[index] = oldValues[i]; + } + } + } +} diff --git a/core/src/main/java/io/questdb/client/std/Chars.java b/core/src/main/java/io/questdb/client/std/Chars.java index b3dd42e..b3ded7f 100644 --- a/core/src/main/java/io/questdb/client/std/Chars.java +++ b/core/src/main/java/io/questdb/client/std/Chars.java @@ -51,10 +51,6 @@ public static void base64Encode(@Nullable BinarySequence sequence, int maxLength } } - public static boolean contains(@NotNull CharSequence sequence, @NotNull CharSequence term) { - return indexOf(sequence, 0, sequence.length(), term) != -1; - } - public static boolean equals(@NotNull CharSequence l, @NotNull CharSequence r) { if (l == r) { return true; @@ -356,18 +352,6 @@ public static int lowerCaseHashCode(CharSequence value) { return h; } - public static boolean noMatch(CharSequence l, int llo, int lhi, CharSequence r, int rlo, int rhi) { - int lp = llo; - int rp = rlo; - while (lp < lhi && rp < rhi) { - if (Character.toLowerCase(l.charAt(lp++)) != r.charAt(rp++)) { - return true; - } - - } - return lp != lhi || rp != rhi; - } - public static boolean startsWith(@Nullable CharSequence cs, @Nullable CharSequence starts) { if (cs == null || starts == null) { return false; @@ -380,6 +364,22 @@ public static boolean startsWith(CharSequence _this, char c) { return _this.length() > 0 && _this.charAt(0) == c; } + public static String toLowerCase(@Nullable CharSequence value) { + if (value == null) { + return null; + } + final int len = value.length(); + if (len == 0) { + return ""; + } + + final Utf16Sink b = Misc.getThreadLocalSink(); + for (int i = 0; i < len; i++) { + b.put(Character.toLowerCase(value.charAt(i))); + } + return b.toString(); + } + public static String toLowerCaseAscii(@Nullable CharSequence value) { if (value == null) { return null; diff --git a/core/src/main/java/io/questdb/client/std/ConcurrentHashMap.java b/core/src/main/java/io/questdb/client/std/ConcurrentHashMap.java deleted file mode 100644 index 6b29bba..0000000 --- a/core/src/main/java/io/questdb/client/std/ConcurrentHashMap.java +++ /dev/null @@ -1,3791 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.std; - -/* - * ORACLE PROPRIETARY/CONFIDENTIAL. Use is subject to license terms. - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - */ - -/* - * - * - * - * - * - * Written by Doug Lea with assistance from members of JCP JSR-166 - * Expert Group and released to the public domain, as explained at - * http://creativecommons.org/publicdomain/zero/1.0/ - */ - -import io.questdb.client.std.str.CloneableMutable; -import org.jetbrains.annotations.NotNull; - -import java.io.ObjectStreamField; -import java.io.Serializable; -import java.lang.ThreadLocal; -import java.lang.reflect.ParameterizedType; -import java.lang.reflect.Type; -import java.util.AbstractMap; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.Hashtable; -import java.util.Iterator; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Set; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.LockSupport; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.BiFunction; -import java.util.function.Function; - -/** - * A hash table supporting full concurrency of retrievals and - * high expected concurrency for updates. This class obeys the - * same functional specification as {@link java.util.Hashtable}, and - * includes versions of methods corresponding to each method of - * {@code Hashtable}. However, even though all operations are - * thread-safe, retrieval operations do not entail locking, - * and there is not any support for locking the entire table - * in a way that prevents all access. This class is fully - * interoperable with {@code Hashtable} in programs that rely on its - * thread safety but not on its synchronization details. - *

    Retrieval operations (including {@code get}) generally do not - * block, so may overlap with update operations (including {@code put} - * and {@code remove}). Retrievals reflect the results of the most - * recently completed update operations holding upon their - * onset. (More formally, an update operation for a given key bears a - * happens-before relation with any (non-null) retrieval for - * that key reporting the updated value.) For aggregate operations - * such as {@code putAll} and {@code clear}, concurrent retrievals may - * reflect insertion or removal of only some entries. Similarly, - * Iterators, Spliterators and Enumerations return elements reflecting the - * state of the hash table at some point at or since the creation of the - * iterator/enumeration. They do not throw {@link - * java.util.ConcurrentModificationException ConcurrentModificationException}. - * However, iterators are designed to be used by only one thread at a time. - * Bear in mind that the results of aggregate status methods including - * {@code size}, {@code isEmpty}, and {@code containsValue} are typically - * useful only when a map is not undergoing concurrent updates in other threads. - * Otherwise the results of these methods reflect transient states - * that may be adequate for monitoring or estimation purposes, but not - * for program control. - *

    The table is dynamically expanded when there are too many - * collisions (i.e., keys that have distinct hash codes but fall into - * the same slot modulo the table size), with the expected average - * effect of maintaining roughly two bins per mapping (corresponding - * to a 0.75 load factor threshold for resizing). There may be much - * variance around this average as mappings are added and removed, but - * overall, this maintains a commonly accepted time/space tradeoff for - * hash tables. However, resizing this or any other kind of hash - * table may be a relatively slow operation. When possible, it is a - * good idea to provide a size estimate as an optional {@code - * initialCapacity} constructor argument. An additional optional - * {@code loadFactor} constructor argument provides a further means of - * customizing initial table capacity by specifying the table density - * to be used in calculating the amount of space to allocate for the - * given number of elements. Also, for compatibility with previous - * versions of this class, constructors may optionally specify an - * expected {@code concurrencyLevel} as an additional hint for - * internal sizing. Note that using many keys with exactly the same - * {@code hashCode()} is a sure way to slow down performance of any - * hash table. To ameliorate impact, when keys are {@link Comparable}, - * this class may use comparison order among keys to help break ties. - *

    A {@link Set} projection of a ConcurrentHashMap may be created - * (using {@link #newKeySet()} or {@link #newKeySet(int)}), or viewed - * (using {@link #keySet(Object)} when only keys are of interest, and the - * mapped values are (perhaps transiently) not used or all take the - * same mapping value. - *

    This class and its views and iterators implement all of the - * optional methods of the {@link Map} and {@link Iterator} - * interfaces. - *

    Like {@link Hashtable} but unlike {@link HashMap}, this class - * does not allow {@code null} to be used as a key or value. - *

    ConcurrentHashMaps support a set of sequential and parallel bulk - * operations that are designed - * to be safely, and often sensibly, applied even with maps that are - * being concurrently updated by other threads; for example, when - * computing a snapshot summary of the values in a shared registry. - * There are three kinds of operation, each with four forms, accepting - * functions with Keys, Values, Entries, and (Key, Value) arguments - * and/or return values. Because the elements of a ConcurrentHashMap - * are not ordered in any particular way, and may be processed in - * different orders in different parallel executions, the correctness - * of supplied functions should not depend on any ordering, or on any - * other objects or values that may transiently change while - * computation is in progress; and except for forEach actions, should - * ideally be side-effect-free. Bulk operations on {@link java.util.Map.Entry} - * objects do not support method {@code setValue}. - *

      - *
    • forEach: Perform a given action on each element. - * A variant form applies a given transformation on each element - * before performing the action.
      • - *
    • search: Return the first available non-null result of - * applying a given function on each element; skipping further - * search when a result is found.
      • - *
    • reduce: Accumulate each element. The supplied reduction - * function cannot rely on ordering (more formally, it should be - * both associative and commutative). There are five variants: - *
        - *
      • Plain reductions. (There is not a form of this method for - * (key, value) function arguments since there is no corresponding - * return type.)
        • - *
      • Mapped reductions that accumulate the results of a given - * function applied to each element.
        • - *
      • Reductions to scalar doubles, longs, and ints, using a - * given basis value.
        • - *
      - *
      • - *
    - *

    The concurrency properties of bulk operations follow - * from those of ConcurrentHashMap: Any non-null result returned - * from {@code get(key)} and related access methods bears a - * happens-before relation with the associated insertion or - * update. The result of any bulk operation reflects the - * composition of these per-element relations (but is not - * necessarily atomic with respect to the map as a whole unless it - * is somehow known to be quiescent). Conversely, because keys - * and values in the map are never null, null serves as a reliable - * atomic indicator of the current lack of any result. To - * maintain this property, null serves as an implicit basis for - * all non-scalar reduction operations. For the double, long, and - * int versions, the basis should be one that, when combined with - * any other value, returns that other value (more formally, it - * should be the identity element for the reduction). Most common - * reductions have these properties; for example, computing a sum - * with basis 0 or a minimum with basis MAX_VALUE. - *

    Search and transformation functions provided as arguments - * should similarly return null to indicate the lack of any result - * (in which case it is not used). In the case of mapped - * reductions, this also enables transformations to serve as - * filters, returning null (or, in the case of primitive - * specializations, the identity basis) if the element should not - * be combined. You can create compound transformations and - * filterings by composing them yourself under this "null means - * there is nothing there now" rule before using them in search or - * reduce operations. - *

    Methods accepting and/or returning Entry arguments maintain - * key-value associations. They may be useful for example when - * finding the key for the greatest value. Note that "plain" Entry - * arguments can be supplied using {@code new - * AbstractMap.SimpleEntry(k,v)}. - *

    Bulk operations may complete abruptly, throwing an - * exception encountered in the application of a supplied - * function. Bear in mind when handling such exceptions that other - * concurrently executing functions could also have thrown - * exceptions, or would have done so if the first exception had - * not occurred. - *

    Speedups for parallel compared to sequential forms are common - * but not guaranteed. Parallel operations involving brief functions - * on small maps may execute more slowly than sequential forms if the - * underlying work to parallelize the computation is more expensive - * than the computation itself. Similarly, parallelization may not - * lead to much actual parallelism if all processors are busy - * performing unrelated tasks. - *

    All arguments to all task methods must be non-null. - *

    This class is a member of the - * - * Java Collections Framework. - * - * @param the type of mapped values - * @author Doug Lea - * @since 1.5 - */ -@SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter") -public class ConcurrentHashMap extends AbstractMap - implements ConcurrentMap, Serializable { - static final int HASH_BITS = 0x7fffffff; // usable bits of normal node hash - - /* - * Overview: - * - * The primary design goal of this hash table is to maintain - * concurrent readability (typically method get(), but also - * iterators and related methods) while minimizing update - * contention. Secondary goals are to keep space consumption about - * the same or better than java.util.HashMap, and to support high - * initial insertion rates on an empty table by many threads. - * - * This map usually acts as a binned (bucketed) hash table. Each - * key-value mapping is held in a Node. Most nodes are instances - * of the basic Node class with hash, key, value, and next - * fields. However, various subclasses exist: TreeNodes are - * arranged in balanced trees, not lists. TreeBins hold the roots - * of sets of TreeNodes. ForwardingNodes are placed at the heads - * of bins during resizing. ReservationNodes are used as - * placeholders while establishing values in computeIfAbsent and - * related methods. The types TreeBin, ForwardingNode, and - * ReservationNode do not hold normal user keys, values, or - * hashes, and are readily distinguishable during search etc - * because they have negative hash fields and null key and value - * fields. (These special nodes are either uncommon or transient, - * so the impact of carrying around some unused fields is - * insignificant.) - * - * The table is lazily initialized to a power-of-two size upon the - * first insertion. Each bin in the table normally contains a - * list of Nodes (most often, the list has only zero or one Node). - * Table accesses require volatile/atomic reads, writes, and - * CASes. Because there is no other way to arrange this without - * adding further indirections, we use intrinsics - * (sun.misc.Unsafe) operations. - * - * We use the top (sign) bit of Node hash fields for control - * purposes -- it is available anyway because of addressing - * constraints. Nodes with negative hash fields are specially - * handled or ignored in map methods. - * - * Insertion (via put or its variants) of the first node in an - * empty bin is performed by just CASing it to the bin. This is - * by far the most common case for put operations under most - * key/hash distributions. Other update operations (insert, - * delete, and replace) require locks. We do not want to waste - * the space required to associate a distinct lock object with - * each bin, so instead use the first node of a bin list itself as - * a lock. Locking support for these locks relies on builtin - * "synchronized" monitors. - * - * Using the first node of a list as a lock does not by itself - * suffice though: When a node is locked, any update must first - * validate that it is still the first node after locking it, and - * retry if not. Because new nodes are always appended to lists, - * once a node is first in a bin, it remains first until deleted - * or the bin becomes invalidated (upon resizing). - * - * The main disadvantage of per-bin locks is that other update - * operations on other nodes in a bin list protected by the same - * lock can stall, for example when user equals() or mapping - * functions take a long time. However, statistically, under - * random hash codes, this is not a common problem. Ideally, the - * frequency of nodes in bins follows a Poisson distribution - * (http://en.wikipedia.org/wiki/Poisson_distribution) with a - * parameter of about 0.5 on average, given the resizing threshold - * of 0.75, although with a large variance because of resizing - * granularity. Ignoring variance, the expected occurrences of - * list size k are (exp(-0.5) * pow(0.5, k) / factorial(k)). The - * first values are: - * - * 0: 0.60653066 - * 1: 0.30326533 - * 2: 0.07581633 - * 3: 0.01263606 - * 4: 0.00157952 - * 5: 0.00015795 - * 6: 0.00001316 - * 7: 0.00000094 - * 8: 0.00000006 - * more: less than 1 in ten million - * - * Lock contention probability for two threads accessing distinct - * elements is roughly 1 / (8 * #elements) under random hashes. - * - * Actual hash code distributions encountered in practice - * sometimes deviate significantly from uniform randomness. This - * includes the case when N > (1<<30), so some keys MUST collide. - * Similarly for dumb or hostile usages in which multiple keys are - * designed to have identical hash codes or ones that differs only - * in masked-out high bits. So we use a secondary strategy that - * applies when the number of nodes in a bin exceeds a - * threshold. These TreeBins use a balanced tree to hold nodes (a - * specialized form of red-black trees), bounding search time to - * O(log N). Each search step in a TreeBin is at least twice as - * slow as in a regular list, but given that N cannot exceed - * (1<<64) (before running out of addresses) this bounds search - * steps, lock hold times, etc, to reasonable constants (roughly - * 100 nodes inspected per operation worst case) so long as keys - * are Comparable (which is very common -- String, Long, etc). - * TreeBin nodes (TreeNodes) also maintain the same "next" - * traversal pointers as regular nodes, so can be traversed in - * iterators in the same way. - * - * The table is resized when occupancy exceeds a percentage - * threshold (nominally, 0.75, but see below). Any thread - * noticing an overfull bin may assist in resizing after the - * initiating thread allocates and sets up the replacement array. - * However, rather than stalling, these other threads may proceed - * with insertions etc. The use of TreeBins shields us from the - * worst case effects of overfilling while resizes are in - * progress. Resizing proceeds by transferring bins, one by one, - * from the table to the next table. However, threads claim small - * blocks of indices to transfer (via field transferIndex) before - * doing so, reducing contention. A generation stamp in field - * sizeCtl ensures that resizings do not overlap. Because we are - * using power-of-two expansion, the elements from each bin must - * either stay at same index, or move with a power of two - * offset. We eliminate unnecessary node creation by catching - * cases where old nodes can be reused because their next fields - * won't change. On average, only about one-sixth of them need - * cloning when a table doubles. The nodes they replace will be - * garbage collectable as soon as they are no longer referenced by - * any reader thread that may be in the midst of concurrently - * traversing table. Upon transfer, the old table bin contains - * only a special forwarding node (with hash field "MOVED") that - * contains the next table as its key. On encountering a - * forwarding node, access and update operations restart, using - * the new table. - * - * Each bin transfer requires its bin lock, which can stall - * waiting for locks while resizing. However, because other - * threads can join in and help resize rather than contend for - * locks, average aggregate waits become shorter as resizing - * progresses. The transfer operation must also ensure that all - * accessible bins in both the old and new table are usable by any - * traversal. This is arranged in part by proceeding from the - * last bin (table.length - 1) up towards the first. Upon seeing - * a forwarding node, traversals (see class Traverser) arrange to - * move to the new table without revisiting nodes. To ensure that - * no intervening nodes are skipped even when moved out of order, - * a stack (see class TableStack) is created on first encounter of - * a forwarding node during a traversal, to maintain its place if - * later processing the current table. The need for these - * save/restore mechanics is relatively rare, but when one - * forwarding node is encountered, typically many more will be. - * So Traversers use a simple caching scheme to avoid creating so - * many new TableStack nodes. (Thanks to Peter Levart for - * suggesting use of a stack here.) - * - * The traversal scheme also applies to partial traversals of - * ranges of bins (via an alternate Traverser constructor) - * to support partitioned aggregate operations. Also, read-only - * operations give up if ever forwarded to a null table, which - * provides support for shutdown-style clearing, which is also not - * currently implemented. - * - * Lazy table initialization minimizes footprint until first use, - * and also avoids resizings when the first operation is from a - * putAll, constructor with map argument, or deserialization. - * These cases attempt to override the initial capacity settings, - * but harmlessly fail to take effect in cases of races. - * - * The element count is maintained using a specialization of - * LongAdder. We need to incorporate a specialization rather than - * just use a LongAdder in order to access implicit - * contention-sensing that leads to creation of multiple - * CounterCells. The counter mechanics avoid contention on - * updates but can encounter cache thrashing if read too - * frequently during concurrent access. To avoid reading so often, - * resizing under contention is attempted only upon adding to a - * bin already holding two or more nodes. Under uniform hash - * distributions, the probability of this occurring at threshold - * is around 13%, meaning that only about 1 in 8 puts check - * threshold (and after resizing, many fewer do so). - * - * TreeBins use a special form of comparison for search and - * related operations (which is the main reason we cannot use - * existing collections such as TreeMaps). TreeBins contain - * Comparable elements, but may contain others, as well as - * elements that are Comparable but not necessarily Comparable for - * the same T, so we cannot invoke compareTo among them. To handle - * this, the tree is ordered primarily by hash value, then by - * Comparable.compareTo order if applicable. On lookup at a node, - * if elements are not comparable or compare as 0 then both left - * and right children may need to be searched in the case of tied - * hash values. (This corresponds to the full list search that - * would be necessary if all elements were non-Comparable and had - * tied hashes.) On insertion, to keep a total ordering (or as - * close as is required here) across rebalancings, we compare - * classes and identityHashCodes as tie-breakers. The red-black - * balancing code is updated from pre-jdk-collections - * (http://gee.cs.oswego.edu/dl/classes/collections/RBCell.java) - * based in turn on Cormen, Leiserson, and Rivest "Introduction to - * Algorithms" (CLR). - * - * TreeBins also require an additional locking mechanism. While - * list traversal is always possible by readers even during - * updates, tree traversal is not, mainly because of tree-rotations - * that may change the root node and/or its linkages. TreeBins - * include a simple read-write lock mechanism parasitic on the - * main bin-synchronization strategy: Structural adjustments - * associated with an insertion or removal are already bin-locked - * (and so cannot conflict with other writers) but must wait for - * ongoing readers to finish. Since there can be only one such - * waiter, we use a simple scheme using a single "waiter" field to - * block writers. However, readers need never block. If the root - * lock is held, they proceed along the slow traversal path (via - * next-pointers) until the lock becomes available or the list is - * exhausted, whichever comes first. These cases are not fast, but - * maximize aggregate expected throughput. - * - * Maintaining API and serialization compatibility with previous - * versions of this class introduces several oddities. Mainly: We - * leave untouched but unused constructor arguments referring to - * concurrencyLevel. We accept a loadFactor constructor argument, - * but apply it only to initial table capacity (which is the only - * time that we can guarantee to honor it.) We also declare an - * unused "Segment" class that is instantiated in minimal form - * only when serializing. - * - * Also, solely for compatibility with previous versions of this - * class, it extends AbstractMap, even though all of its methods - * are overridden, so it is just useless baggage. - * - * This file is organized to make things a little easier to follow - * while reading than they might otherwise: First the main static - * declarations and utilities, then fields, then main public - * methods (with a few factorings of multiple public methods into - * internal ones), then sizing methods, trees, traversers, and - * bulk operations. - */ - - /* ---------------- Constants -------------- */ - /** - * The largest possible (non-power of two) array size. - * Needed by toArray and related methods. - */ - static final int MAX_ARRAY_SIZE = Integer.MAX_VALUE - 8; - /** - * The smallest table capacity for which bins may be treeified. - * (Otherwise the table is resized if too many nodes in a bin.) - * The value should be at least 4 * TREEIFY_THRESHOLD to avoid - * conflicts between resizing and treeification thresholds. - */ - static final int MIN_TREEIFY_CAPACITY = 64; - /* - * Encodings for Node hash fields. See above for explanation. - */ - static final int MOVED = -1; // hash for forwarding nodes - /** - * Number of CPUS, to place bounds on some sizings - */ - static final int NCPU = Runtime.getRuntime().availableProcessors(); - static final int RESERVED = -3; // hash for transient reservations - static final int TREEBIN = -2; // hash for roots of trees - /** - * The bin count threshold for using a tree rather than list for a - * bin. Bins are converted to trees when adding an element to a - * bin with at least this many nodes. The value must be greater - * than 2, and should be at least 8 to mesh with assumptions in - * tree removal about conversion back to plain bins upon - * shrinkage. - */ - static final int TREEIFY_THRESHOLD = 8; - /** - * The bin count threshold for untreeifying a (split) bin during a - * resize operation. Should be less than TREEIFY_THRESHOLD, and at - * most 6 to mesh with shrinkage detection under removal. - */ - static final int UNTREEIFY_THRESHOLD = 6; - /* ---------------- Fields -------------- */ - private static final long ABASE; - private static final int ASHIFT; - /* - * Volatile access methods are used for table elements as well as - * elements of in-progress next table while resizing. All uses of - * the tab arguments must be null checked by callers. All callers - * also paranoically precheck that tab's length is not zero (or an - * equivalent check), thus ensuring that any index argument taking - * the form of a hash value anded with (length - 1) is a valid - * index. Note that, to be correct wrt arbitrary concurrency - * errors by users, these checks must operate on local variables, - * which accounts for some odd-looking inline assignments below. - * Note that calls to setTabAt always occur within locked regions, - * and so in principle require only release ordering, not - * full volatile semantics, but are currently coded as volatile - * writes to be conservative. - */ - private static final long BASECOUNT; - private static final long CELLSBUSY; - private static final long CELLVALUE; - /** - * The default initial table capacity. Must be a power of 2 - * (i.e., at least 1) and at most MAXIMUM_CAPACITY. - */ - private static final int DEFAULT_CAPACITY = 16; - /** - * The load factor for this table. Overrides of this value in - * constructors affect only the initial table capacity. The - * actual floating point value isn't normally used -- it is - * simpler to use expressions such as {@code n - (n >>> 2)} for - * the associated resizing threshold. - */ - private static final float LOAD_FACTOR = 0.75f; - /** - * The largest possible table capacity. This value must be - * exactly 1<<30 to stay within Java array allocation and indexing - * bounds for power of two table sizes, and is further required - * because the top two bits of 32bit hash fields are used for - * control purposes. - */ - private static final int MAXIMUM_CAPACITY = 1 << 30; - /** - * Minimum number of rebinnings per transfer step. Ranges are - * subdivided to allow multiple resizer threads. This value - * serves as a lower bound to avoid resizers encountering - * excessive memory contention. The value should be at least - * DEFAULT_CAPACITY. - */ - private static final int MIN_TRANSFER_STRIDE = 16; - private static final long PROBE; - - /* ---------------- Nodes -------------- */ - /** - * The increment for generating probe values - */ - private static final int PROBE_INCREMENT = 0x9e3779b9; - - /* ---------------- Static utilities -------------- */ - /** - * The number of bits used for generation stamp in sizeCtl. - * Must be at least 6 for 32bit arrays. - */ - private static final int RESIZE_STAMP_BITS = 16; - /** - * The maximum number of threads that can help resize. - * Must fit in 32 - RESIZE_STAMP_BITS bits. - */ - private static final int MAX_RESIZERS = (1 << (32 - RESIZE_STAMP_BITS)) - 1; - /** - * The bit shift for recording size stamp in sizeCtl. - */ - private static final int RESIZE_STAMP_SHIFT = 32 - RESIZE_STAMP_BITS; - private static final long SEED; - - /* ---------------- Table element access -------------- */ - /** - * The increment of seeder per new instance - */ - private static final long SEEDER_INCREMENT = 0xbb67ae8584caa73bL; - private static final long SIZECTL; - private static final long TRANSFERINDEX; - /** - * Generates per-thread initialization/probe field - */ - private static final AtomicInteger probeGenerator = new AtomicInteger(); - /** - * The next seed for default constructors. - */ - private static final AtomicLong seeder = new AtomicLong(initialSeed()); - /** - * For serialization compatibility. - */ - private static final ObjectStreamField[] serialPersistentFields = { - new ObjectStreamField("segments", Segment[].class), - new ObjectStreamField("segmentMask", Integer.TYPE), - new ObjectStreamField("segmentShift", Integer.TYPE) - }; - private static final long serialVersionUID = 7249069246763182397L; - private final java.lang.ThreadLocal> tlTraverser = ThreadLocal.withInitial(Traverser::new); - /** - * The array of bins. Lazily initialized upon first insertion. - * Size is always a power of two. Accessed directly by iterators. - */ - transient volatile Node[] table; - /** - * Base counter value, used mainly when there is no contention, - * but also as a fallback during table initialization - * races. Updated via CAS. - */ - private transient volatile long baseCount; - /** - * Spinlock (locked via CAS) used when resizing and/or creating CounterCells. - */ - private transient volatile int cellsBusy; - /** - * Table of counter cells. When non-null, size is a power of 2. - */ - private transient volatile CounterCell[] counterCells; - // Original (since JDK1.2) Map methods - private transient EntrySetView entrySet; - private transient boolean ics = true; - /* ---------------- Public operations -------------- */ - // views - private transient KeySetView keySet; - /** - * The next table to use; non-null only while resizing. - */ - private transient volatile Node[] nextTable; - /** - * Table initialization and resizing control. When negative, the - * table is being initialized or resized: -1 for initialization, - * else -(1 + the number of active resizing threads). Otherwise, - * when table is null, holds the initial table size to use upon - * creation, or 0 for default. After initialization, holds the - * next element count value upon which to resize the table. - */ - private transient volatile int sizeCtl; - /** - * The next table index (plus one) to split while resizing. - */ - private transient volatile int transferIndex; - private transient ValuesView values; - - /** - * Creates a new, empty map with the default initial table size (16). - */ - public ConcurrentHashMap(boolean isCaseSensitive) { - this.ics = isCaseSensitive; - } - - public ConcurrentHashMap() { - this(true); - } - - /** - * Creates a new, empty map with an initial table size - * accommodating the specified number of elements without the need - * to dynamically resize. - * - * @param initialCapacity The implementation performs internal - * sizing to accommodate this many elements. - * @throws IllegalArgumentException if the initial capacity of - * elements is negative - */ - public ConcurrentHashMap(int initialCapacity, boolean isCaseSensitive) { - this.ics = isCaseSensitive; - if (initialCapacity < 0) - throw new IllegalArgumentException(); - this.sizeCtl = ((initialCapacity >= (MAXIMUM_CAPACITY >>> 1)) ? - MAXIMUM_CAPACITY : - tableSizeFor(initialCapacity + (initialCapacity >>> 1) + 1)); - } - - public ConcurrentHashMap(int initialCapacity) { - this(initialCapacity, true); - } - - /** - * Creates a new map with the same mappings as the given map. - * - * @param m the map - */ - public ConcurrentHashMap(Map m, boolean isCaseSensitive) { - this.ics = isCaseSensitive; - this.sizeCtl = DEFAULT_CAPACITY; - putAll(m); - } - - public ConcurrentHashMap(Map m) { - this(m, true); - } - - /** - * Creates a new, empty map with an initial table size based on - * the given number of elements ({@code initialCapacity}), table - * density ({@code loadFactor}), and number of concurrently - * updating threads ({@code concurrencyLevel}). - * - * @param initialCapacity the initial capacity. The implementation - * performs internal sizing to accommodate this many elements, - * given the specified load factor. - * @param loadFactor the load factor (table density) for - * establishing the initial table size - * @throws IllegalArgumentException if the initial capacity is - * negative or the load factor or concurrencyLevel are - * nonpositive - */ - public ConcurrentHashMap(int initialCapacity, float loadFactor, boolean isCaseSensitive) { - this.ics = isCaseSensitive; - if (!(loadFactor > 0.0f) || initialCapacity < 0) - throw new IllegalArgumentException(); - if (initialCapacity < 1) // Use at least as many bins - initialCapacity = 1; // as estimated threads - long size = (long) (1.0 + (long) initialCapacity / loadFactor); - this.sizeCtl = (size >= (long) MAXIMUM_CAPACITY) ? - MAXIMUM_CAPACITY : tableSizeFor((int) size); - } - - public ConcurrentHashMap(int initialCapacity, float loadFactor) { - this(initialCapacity, loadFactor, true); - } - - /** - * Creates a new {@link Set} backed by a ConcurrentHashMap - * from the given type to {@code Boolean.TRUE}. - * - * @return the new set - * @since 1.8 - */ - public static KeySetView newKeySet(boolean isCaseSensitive) { - return new KeySetView<>(new ConcurrentHashMap<>(isCaseSensitive), Boolean.TRUE); - } - - public static KeySetView newKeySet() { - return new KeySetView<>(new ConcurrentHashMap<>(), Boolean.TRUE); - } - - /** - * Creates a new {@link Set} backed by a ConcurrentHashMap - * from the given type to {@code Boolean.TRUE}. - * - * @param initialCapacity The implementation performs internal - * sizing to accommodate this many elements. - * @return the new set - * @throws IllegalArgumentException if the initial capacity of - * elements is negative - * @since 1.8 - */ - public static KeySetView newKeySet(int initialCapacity, boolean isCaseSensitive) { - return new KeySetView<>(new ConcurrentHashMap<>(initialCapacity, isCaseSensitive), Boolean.TRUE); - } - - public static KeySetView newKeySet(int initialCapacity) { - return new KeySetView<>(new ConcurrentHashMap<>(initialCapacity), Boolean.TRUE); - } - - /** - * Removes all of the mappings from this map. - */ - public void clear() { - long delta = 0L; // negative number of deletions - int i = 0; - Node[] tab = table; - while (tab != null && i < tab.length) { - int fh; - Node f = tabAt(tab, i); - if (f == null) - ++i; - else if ((fh = f.hash) == MOVED) { - tab = helpTransfer(tab, f); - i = 0; // restart - } else { - synchronized (f) { - if (tabAt(tab, i) == f) { - Node p = (fh >= 0 ? f : - (f instanceof TreeBin) ? - ((TreeBin) f).first : null); - while (p != null) { - --delta; - p = p.next; - } - setTabAt(tab, i++, null); - } - } - } - } - if (delta != 0L) - addCount(delta, -1); - } - - /** - * Attempts to compute a mapping for the specified key and its - * current mapped value (or {@code null} if there is no current - * mapping). The entire method invocation is performed atomically. - * Some attempted update operations on this map by other threads - * may be blocked while computation is in progress, so the - * computation should be short and simple, and must not attempt to - * update any other mappings of this Map. - * - * @param key key with which the specified value is to be associated - * @param remappingFunction the function to compute a value - * @return the new value associated with the specified key, or null if none - * @throws NullPointerException if the specified key or remappingFunction - * is null - * @throws IllegalStateException if the computation detectably - * attempts a recursive update to this map that would - * otherwise never complete - * @throws RuntimeException or Error if the remappingFunction does so, - * in which case the mapping is unchanged - */ - public V compute(CharSequence key, BiFunction remappingFunction) { - if (key == null || remappingFunction == null) - throw new NullPointerException(); - int h = spread(keyHashCode(key)); - V val = null; - int delta = 0; - int binCount = 0; - for (Node[] tab = table; ; ) { - Node f; - int n, i, fh; - if (tab == null || (n = tab.length) == 0) - tab = initTable(); - else if ((f = tabAt(tab, i = (n - 1) & h)) == null) { - Node r = new ReservationNode<>(ics); - synchronized (r) { - if (casTabAt(tab, i, r)) { - binCount = 1; - Node node = null; - try { - if ((val = remappingFunction.apply(key, null)) != null) { - delta = 1; - node = new Node<>(h, maybeCopyKey(key), val, null, ics); - } - } finally { - setTabAt(tab, i, node); - } - } - } - if (binCount != 0) - break; - } else if ((fh = f.hash) == MOVED) - tab = helpTransfer(tab, f); - else { - synchronized (f) { - if (tabAt(tab, i) == f) { - if (fh >= 0) { - binCount = 1; - for (Node e = f, pred = null; ; ++binCount) { - CharSequence ek; - if (e.hash == h && - ((ek = e.key) == key || (keyEquals(key, ek)))) { - val = remappingFunction.apply(key, e.val); - if (val != null) - e.val = val; - else { - delta = -1; - Node en = e.next; - if (pred != null) - pred.next = en; - else - setTabAt(tab, i, en); - } - break; - } - pred = e; - if ((e = e.next) == null) { - val = remappingFunction.apply(key, null); - if (val != null) { - delta = 1; - pred.next = - new Node<>(h, maybeCopyKey(key), val, null, ics); - } - break; - } - } - } else if (f instanceof TreeBin) { - binCount = 1; - TreeBin t = (TreeBin) f; - TreeNode r, p; - if ((r = t.root) != null) - p = r.findTreeNode(h, key, null); - else - p = null; - V pv = (p == null) ? null : p.val; - val = remappingFunction.apply(key, pv); - if (val != null) { - if (p != null) - p.val = val; - else { - delta = 1; - t.putTreeVal(h, key, val); - } - } else if (p != null) { - delta = -1; - if (t.removeTreeNode(p)) - setTabAt(tab, i, untreeify(t.first)); - } - } - } - } - if (binCount != 0) { - if (binCount >= TREEIFY_THRESHOLD) - treeifyBin(tab, i); - break; - } - } - } - if (delta != 0) - addCount(delta, binCount); - return val; - } - - /** - * If the specified key is not already associated with a value, - * attempts to compute its value using the given mapping function - * and enters it into this map unless {@code null}. The entire - * method invocation is performed atomically, so the function is - * applied at most once per key. Some attempted update operations - * on this map by other threads may be blocked while computation - * is in progress, so the computation should be short and simple, - * and must not attempt to update any other mappings of this map. - * - * @param key key with which the specified value is to be associated - * @param token token to pass to the mapping function - * @param mappingFunction the function to compute a value - * @return the current (existing or computed) value associated with - * the specified key, or null if the computed value is null - * @throws NullPointerException if the specified key or mappingFunction - * is null - * @throws IllegalStateException if the computation detectably - * attempts a recursive update to this map that would - * otherwise never complete - * @throws RuntimeException or Error if the mappingFunction does so, - * in which case the mapping is left unestablished - */ - public V computeIfAbsent(CharSequence key, Object token, BiFunction mappingFunction) { - if (key == null || mappingFunction == null) - throw new NullPointerException(); - int h = spread(keyHashCode(key)); - V val = null; - int binCount = 0; - for (Node[] tab = table; ; ) { - Node f; - int n, i, fh; - if (tab == null || (n = tab.length) == 0) - tab = initTable(); - else if ((f = tabAt(tab, i = (n - 1) & h)) == null) { - Node r = new ReservationNode<>(ics); - synchronized (r) { - if (casTabAt(tab, i, r)) { - binCount = 1; - Node node = null; - try { - if ((val = mappingFunction.apply(key, token)) != null) - node = new Node<>(h, maybeCopyKey(key), val, null, ics); - } finally { - setTabAt(tab, i, node); - } - } - } - if (binCount != 0) - break; - } else if ((fh = f.hash) == MOVED) - tab = helpTransfer(tab, f); - else { - boolean added = false; - synchronized (f) { - if (tabAt(tab, i) == f) { - if (fh >= 0) { - binCount = 1; - for (Node e = f; ; ++binCount) { - CharSequence ek; - if (e.hash == h && - ((ek = e.key) == key || (keyEquals(key, ek)))) { - val = e.val; - break; - } - Node pred = e; - if ((e = e.next) == null) { - if ((val = mappingFunction.apply(key, token)) != null) { - added = true; - pred.next = new Node(h, key, val, null, ics); - } - break; - } - } - } else if (f instanceof TreeBin) { - binCount = 2; - TreeBin t = (TreeBin) f; - TreeNode r, p; - if ((r = t.root) != null && - (p = r.findTreeNode(h, key, null)) != null) - val = p.val; - else if ((val = mappingFunction.apply(key, token)) != null) { - added = true; - t.putTreeVal(h, key, val); - } - } - } - } - if (binCount != 0) { - if (binCount >= TREEIFY_THRESHOLD) - treeifyBin(tab, i); - if (!added) - return val; - break; - } - } - } - if (val != null) - addCount(1L, binCount); - return val; - } - - /** - * If the specified key is not already associated with a value, - * attempts to compute its value using the given mapping function - * and enters it into this map unless {@code null}. The entire - * method invocation is performed atomically, so the function is - * applied at most once per key. Some attempted update operations - * on this map by other threads may be blocked while computation - * is in progress, so the computation should be short and simple, - * and must not attempt to update any other mappings of this map. - * - * @param key key with which the specified value is to be associated - * @param mappingFunction the function to compute a value - * @return the current (existing or computed) value associated with - * the specified key, or null if the computed value is null - * @throws NullPointerException if the specified key or mappingFunction - * is null - * @throws IllegalStateException if the computation detectably - * attempts a recursive update to this map that would - * otherwise never complete - * @throws RuntimeException or Error if the mappingFunction does so, - * in which case the mapping is left unestablished - */ - public V computeIfAbsent(CharSequence key, Function mappingFunction) { - if (key == null || mappingFunction == null) - throw new NullPointerException(); - int h = spread(keyHashCode(key)); - V val = null; - int binCount = 0; - for (Node[] tab = table; ; ) { - Node f; - int n, i, fh; - if (tab == null || (n = tab.length) == 0) - tab = initTable(); - else if ((f = tabAt(tab, i = (n - 1) & h)) == null) { - Node r = new ReservationNode<>(ics); - synchronized (r) { - if (casTabAt(tab, i, r)) { - binCount = 1; - Node node = null; - try { - if ((val = mappingFunction.apply(key)) != null) - node = new Node<>(h, maybeCopyKey(key), val, null, ics); - } finally { - setTabAt(tab, i, node); - } - } - } - if (binCount != 0) - break; - } else if ((fh = f.hash) == MOVED) - tab = helpTransfer(tab, f); - else { - boolean added = false; - synchronized (f) { - if (tabAt(tab, i) == f) { - if (fh >= 0) { - binCount = 1; - for (Node e = f; ; ++binCount) { - CharSequence ek; - if (e.hash == h && - ((ek = e.key) == key || (keyEquals(key, ek)))) { - val = e.val; - break; - } - Node pred = e; - if ((e = e.next) == null) { - if ((val = mappingFunction.apply(key)) != null) { - added = true; - pred.next = new Node<>(h, maybeCopyKey(key), val, null, ics); - } - break; - } - } - } else if (f instanceof TreeBin) { - binCount = 2; - TreeBin t = (TreeBin) f; - TreeNode r, p; - if ((r = t.root) != null && - (p = r.findTreeNode(h, key, null)) != null) - val = p.val; - else if ((val = mappingFunction.apply(key)) != null) { - added = true; - t.putTreeVal(h, key, val); - } - } - } - } - if (binCount != 0) { - if (binCount >= TREEIFY_THRESHOLD) - treeifyBin(tab, i); - if (!added) - return val; - break; - } - } - } - if (val != null) - addCount(1L, binCount); - return val; - } - - /** - * If the value for the specified key is present, attempts to - * compute a new mapping given the key and its current mapped - * value. The entire method invocation is performed atomically. - * Some attempted update operations on this map by other threads - * may be blocked while computation is in progress, so the - * computation should be short and simple, and must not attempt to - * update any other mappings of this map. - * - * @param key key with which a value may be associated - * @param remappingFunction the function to compute a value - * @return the new value associated with the specified key, or null if none - * @throws NullPointerException if the specified key or remappingFunction - * is null - * @throws IllegalStateException if the computation detectably - * attempts a recursive update to this map that would - * otherwise never complete - * @throws RuntimeException or Error if the remappingFunction does so, - * in which case the mapping is unchanged - */ - public V computeIfPresent(CharSequence key, BiFunction remappingFunction) { - if (key == null || remappingFunction == null) - throw new NullPointerException(); - int h = spread(keyHashCode(key)); - V val = null; - int delta = 0; - int binCount = 0; - for (Node[] tab = table; ; ) { - Node f; - int n, i, fh; - if (tab == null || (n = tab.length) == 0) - tab = initTable(); - else if ((f = tabAt(tab, i = (n - 1) & h)) == null) - break; - else if ((fh = f.hash) == MOVED) - tab = helpTransfer(tab, f); - else { - synchronized (f) { - if (tabAt(tab, i) == f) { - if (fh >= 0) { - binCount = 1; - for (Node e = f, pred = null; ; ++binCount) { - CharSequence ek; - if (e.hash == h && - ((ek = e.key) == key || (keyEquals(key, ek)))) { - val = remappingFunction.apply(key, e.val); - if (val != null) - e.val = val; - else { - delta = -1; - Node en = e.next; - if (pred != null) - pred.next = en; - else - setTabAt(tab, i, en); - } - break; - } - pred = e; - if ((e = e.next) == null) - break; - } - } else if (f instanceof TreeBin) { - binCount = 2; - TreeBin t = (TreeBin) f; - TreeNode r, p; - if ((r = t.root) != null && - (p = r.findTreeNode(h, key, null)) != null) { - val = remappingFunction.apply(key, p.val); - if (val != null) - p.val = val; - else { - delta = -1; - if (t.removeTreeNode(p)) - setTabAt(tab, i, untreeify(t.first)); - } - } - } - } - } - if (binCount != 0) - break; - } - } - if (delta != 0) - addCount(delta, binCount); - return val; - } - - /** - * Tests if the specified object is a key in this table. - * - * @param key possible key - * @return {@code true} if and only if the specified object - * is a key in this table, as determined by the - * {@code equals} method; {@code false} otherwise - * @throws NullPointerException if the specified key is null - */ - public boolean containsKey(Object key) { - return get(key) != null; - } - - /** - * Returns {@code true} if this map maps one or more keys to the - * specified value. Note: This method may require a full traversal - * of the map, and is much slower than method {@code containsKey}. - * - * @param value value whose presence in this map is to be tested - * @return {@code true} if this map maps one or more keys to the - * specified value - * @throws NullPointerException if the specified value is null - */ - public boolean containsValue(Object value) { - if (value == null) - throw new NullPointerException(); - Node[] t = table; - if (t != null) { - Traverser it = getTraverser(t); - for (Node p; (p = it.advance()) != null; ) { - V v; - if ((v = p.val) == value || (value.equals(v))) - return true; - } - } - return false; - } - - /** - * Returns a {@link Set} view of the mappings contained in this map. - * The set is backed by the map, so changes to the map are - * reflected in the set, and vice-versa. The set supports element - * removal, which removes the corresponding mapping from the map, - * via the {@code Iterator.remove}, {@code Set.remove}, - * {@code removeAll}, {@code retainAll}, and {@code clear} - * operations. - *

    The view's iterators and spliterators are - * weakly consistent. - * - * @return the set view - */ - @NotNull - public Set> entrySet() { - EntrySetView es; - return (es = entrySet) != null ? es : (entrySet = new EntrySetView<>(this)); - } - - /** - * Compares the specified object with this map for equality. - * Returns {@code true} if the given object is a map with the same - * mappings as this map. This operation may return misleading - * results if either map is concurrently modified during execution - * of this method. - * - * @param o object to be compared for equality with this map - * @return {@code true} if the specified object is equal to this map - */ - public boolean equals(Object o) { - if (o != this) { - if (!(o instanceof Map)) - return false; - Map m = (Map) o; - Traverser it = getTraverser(table); - for (Node p; (p = it.advance()) != null; ) { - V val = p.val; - Object v = m.get(p.key); - if (v == null || (v != val && !v.equals(val))) - return false; - } - for (Map.Entry e : m.entrySet()) { - Object mk, mv, v; - if ((mk = e.getKey()) == null || - (mv = e.getValue()) == null || - (v = get(mk)) == null || - (mv != v && !mv.equals(v))) - return false; - } - } - return true; - } - - /** - * Returns the value to which the specified key is mapped, - * or {@code null} if this map contains no mapping for the key. - *

    More formally, if this map contains a mapping from a key - * {@code k} to a value {@code v} such that {@code key.equals(k)}, - * then this method returns {@code v}; otherwise it returns - * {@code null}. (There can be at most one such mapping.) - * - * @param key map key value - * @return value to which specified key is mapped - * @throws NullPointerException if the specified key is null - */ - @Override - public V get(Object key) { - if (key instanceof CharSequence) { - return get((CharSequence) key); - } - return null; - } - - public V get(CharSequence key) { - Node[] tab; - Node e, p; - int n, eh; - CharSequence ek; - int h = spread(keyHashCode(key)); - if ((tab = table) != null && (n = tab.length) > 0 && - (e = tabAt(tab, (n - 1) & h)) != null) { - if ((eh = e.hash) == h) { - if ((ek = e.key) == key || (ek != null && keyEquals(key, ek))) - return e.val; - } else if (eh < 0) - return (p = e.find(h, key)) != null ? p.val : null; - while ((e = e.next) != null) { - if (e.hash == h && - ((ek = e.key) == key || (ek != null && keyEquals(key, ek)))) - return e.val; - } - } - return null; - } - - /** - * Returns the value to which the specified key is mapped, or the - * given default value if this map contains no mapping for the - * key. - * - * @param key the key whose associated value is to be returned - * @param defaultValue the value to return if this map contains - * no mapping for the given key - * @return the mapping for the key, if present; else the default value - * @throws NullPointerException if the specified key is null - */ - public V getOrDefault(Object key, V defaultValue) { - V v; - return (v = get(key)) == null ? defaultValue : v; - } - - // ConcurrentMap methods - - /** - * Returns the hash code value for this {@link Map}, i.e., - * the sum of, for each key-value pair in the map, - * {@code key.hashCode() ^ value.hashCode()}. - * - * @return the hash code value for this map - */ - public int hashCode() { - int h = 0; - Node[] t = table; - if (t != null) { - Traverser it = getTraverser(t); - for (Node p; (p = it.advance()) != null; ) - h += keyHashCode(p.key) ^ p.val.hashCode(); - } - return h; - } - - /** - * {@inheritDoc} - */ - public boolean isEmpty() { - return sumCount() <= 0L; // ignore transient negative values - } - - /** - * Returns a {@link Set} view of the keys in this map, using the - * given common mapped value for any additions (i.e., {@link - * Collection#add} and {@link Collection#addAll(Collection)}). - * This is of course only appropriate if it is acceptable to use - * the same value for all additions from this view. - * - * @param mappedValue the mapped value to use for any additions - * @return the set view - * @throws NullPointerException if the mappedValue is null - */ - public KeySetView keySet(V mappedValue) { - if (mappedValue == null) - throw new NullPointerException(); - return new KeySetView<>(this, mappedValue); - } - - /** - * Returns a {@link Set} view of the keys contained in this map. - * The set is backed by the map, so changes to the map are - * reflected in the set, and vice-versa. The set supports element - * removal, which removes the corresponding mapping from this map, - * via the {@code Iterator.remove}, {@code Set.remove}, - * {@code removeAll}, {@code retainAll}, and {@code clear} - * operations. It does not support the {@code add} or - * {@code addAll} operations. - *

    The view's iterators and spliterators are - * weakly consistent. - *

    - * - * @return the set view - */ - @NotNull - public KeySetView keySet() { - KeySetView ks; - return (ks = keySet) != null ? ks : (keySet = new KeySetView<>(this, null)); - } - - // Overrides of JDK8+ Map extension method defaults - - /** - * Returns the number of mappings. This method should be used - * instead of {@link #size} because a ConcurrentHashMap may - * contain more mappings than can be represented as an int. The - * value returned is an estimate; the actual count may differ if - * there are concurrent insertions or removals. - * - * @return the number of mappings - * @since 1.8 - */ - public long mappingCount() { - return Math.max(sumCount(), 0L); // ignore transient negative values - } - - /** - * Maps the specified key to the specified value in this table. - * Neither the key nor the value can be null. - *

    The value can be retrieved by calling the {@code get} method - * with a key that is equal to the original key. - * - * @param key key with which the specified value is to be associated - * @param value value to be associated with the specified key - * @return the previous value associated with {@code key}, or - * {@code null} if there was no mapping for {@code key} - * @throws NullPointerException if the specified key or value is null - */ - public V put(CharSequence key, V value) { - return putVal(key, value, false); - } - - /** - * Copies all of the mappings from the specified map to this one. - * These mappings replace any mappings that this map had for any of the - * keys currently in the specified map. - * - * @param m mappings to be stored in this map - */ - public void putAll(@NotNull Map m) { - tryPresize(m.size()); - for (Map.Entry e : m.entrySet()) - putVal(e.getKey(), e.getValue(), false); - } - - /** - * {@inheritDoc} - * - * @return the previous value associated with the specified key, - * or {@code null} if there was no mapping for the key - * @throws NullPointerException if the specified key or value is null - */ - public V putIfAbsent(@NotNull CharSequence key, V value) { - return putVal(key, value, true); - } - - /** - * {@inheritDoc} - * - * @throws NullPointerException if the specified key is null - */ - @SuppressWarnings("unchecked") - public boolean remove(@NotNull Object key, Object value) { - return value != null && replaceNode((CharSequence) key, null, (V) value) != null; - } - // Hashtable legacy methods - - /** - * Removes the key (and its corresponding value) from this map. - * This method does nothing if the key is not in the map. - * - * @param key the key that needs to be removed - * @return the previous value associated with {@code key}, or - * {@code null} if there was no mapping for {@code key} - * @throws NullPointerException if the specified key is null - */ - public V remove(CharSequence key) { - return replaceNode(key, null, null); - } - - // ConcurrentHashMap-only methods - - /** - * {@inheritDoc} - * - * @throws NullPointerException if any of the arguments are null - */ - public boolean replace(@NotNull CharSequence key, @NotNull V oldValue, @NotNull V newValue) { - return replaceNode(key, newValue, oldValue) != null; - } - - /** - * {@inheritDoc} - * - * @return the previous value associated with the specified key, - * or {@code null} if there was no mapping for the key - * @throws NullPointerException if the specified key or value is null - */ - public V replace(@NotNull CharSequence key, @NotNull V value) { - return replaceNode(key, value, null); - } - - /** - * {@inheritDoc} - */ - public int size() { - long n = sumCount(); - return ((n < 0L) ? 0 : - (n > (long) Integer.MAX_VALUE) ? Integer.MAX_VALUE : - (int) n); - } - - /** - * Returns a string representation of this map. The string - * representation consists of a list of key-value mappings (in no - * particular order) enclosed in braces ("{@code {}}"). Adjacent - * mappings are separated by the characters {@code ", "} (comma - * and space). Each key-value mapping is rendered as the key - * followed by an equals sign ("{@code =}") followed by the - * associated value. - * - * @return a string representation of this map - */ - public String toString() { - Traverser it = getTraverser(table); - StringBuilder sb = new StringBuilder(); - sb.append('{'); - Node p; - if ((p = it.advance()) != null) { - for (; ; ) { - CharSequence k = p.key; - V v = p.val; - sb.append(k == this ? "(this Map)" : k); - sb.append('='); - sb.append(v == this ? "(this Map)" : v); - if ((p = it.advance()) == null) - break; - sb.append(',').append(' '); - } - } - return sb.append('}').toString(); - } - - /* ---------------- Special Nodes -------------- */ - - /** - * Returns a {@link Collection} view of the values contained in this map. - * The collection is backed by the map, so changes to the map are - * reflected in the collection, and vice-versa. The collection - * supports element removal, which removes the corresponding - * mapping from this map, via the {@code Iterator.remove}, - * {@code Collection.remove}, {@code removeAll}, - * {@code retainAll}, and {@code clear} operations. It does not - * support the {@code add} or {@code addAll} operations. - *

    The view's iterators and spliterators are - * weakly consistent. - * - * @return the collection view - */ - @NotNull - public Collection values() { - ValuesView vs; - return (vs = values) != null ? vs : (values = new ValuesView<>(this)); - } - - /* ---------------- Table Initialization and Resizing -------------- */ - - private static long initialSeed() { - String pp = System.getProperty("java.util.secureRandomSeed"); - - if (pp != null && pp.equalsIgnoreCase("true")) { - byte[] seedBytes = java.security.SecureRandom.getSeed(8); - long s = (long) (seedBytes[0]) & 0xffL; - for (int i = 1; i < 8; ++i) - s = (s << 8) | ((long) (seedBytes[i]) & 0xffL); - return s; - } - return (mix64(System.currentTimeMillis()) ^ - mix64(System.nanoTime())); - } - - private static boolean keyEquals(final CharSequence lhs, final CharSequence rhs, boolean isCaseSensitive) { - return isCaseSensitive ? Chars.equals(lhs, rhs) : Chars.equalsIgnoreCase(lhs, rhs); - } - - private static int keyHashCode(final CharSequence key, boolean isCaseSensitive) { - return isCaseSensitive ? Chars.hashCode(key) : Chars.lowerCaseHashCode(key); - } - - private static CharSequence maybeCopyKey(CharSequence key) { - return key instanceof CloneableMutable ? ((CloneableMutable) key).copy() : key; - } - - private static long mix64(long z) { - z = (z ^ (z >>> 33)) * 0xff51afd7ed558ccdL; - z = (z ^ (z >>> 33)) * 0xc4ceb9fe1a85ec53L; - return z ^ (z >>> 33); - } - - /** - * Returns a power of two table size for the given desired capacity. - * See Hackers Delight, sec 3.2 - */ - private static int tableSizeFor(int c) { - int n = c - 1; - n |= n >>> 1; - n |= n >>> 2; - n |= n >>> 4; - n |= n >>> 8; - n |= n >>> 16; - return (n < 0) ? 1 : (n >= MAXIMUM_CAPACITY) ? MAXIMUM_CAPACITY : n + 1; - } - - /** - * Adds to count, and if table is too small and not already - * resizing, initiates transfer. If already resizing, helps - * perform transfer if work is available. Rechecks occupancy - * after a transfer to see if another resize is already needed - * because resizings are lagging additions. - * - * @param x the count to add - * @param check if <0, don't check resize, if <= 1 only check if uncontended - */ - private void addCount(long x, int check) { - CounterCell[] as; - long b, s; - if ((as = counterCells) != null || !Unsafe.cas(this, BASECOUNT, b = baseCount, s = b + x)) { - CounterCell a; - long v; - int m; - boolean uncontended = true; - if (as == null || (m = as.length - 1) < 0 || - (a = as[getProbe() & m]) == null || - !(uncontended = Unsafe.cas(a, CELLVALUE, v = a.value, v + x))) { - fullAddCount(x, uncontended); - return; - } - if (check <= 1) - return; - s = sumCount(); - } - if (check >= 0) { - Node[] tab, nt; - int n, sc; - while (s >= (long) (sc = sizeCtl) && (tab = table) != null && - (n = tab.length) < MAXIMUM_CAPACITY) { - int rs = resizeStamp(n); - if (sc < 0) { - if (sc >>> RESIZE_STAMP_SHIFT != rs || sc == rs + MAX_RESIZERS || (nt = nextTable) == null || transferIndex <= 0) - break; - if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc, sc + 1)) - transfer(tab, nt); - } else if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc, - (rs << RESIZE_STAMP_SHIFT) + 2)) - transfer(tab, null); - s = sumCount(); - } - } - } - - // See LongAdder version for explanation - private void fullAddCount(long x, boolean wasUncontended) { - int h; - if ((h = getProbe()) == 0) { - localInit(); // force initialization - h = getProbe(); - wasUncontended = true; - } - boolean collide = false; // True if last slot nonempty - for (; ; ) { - CounterCell[] as; - CounterCell a; - int n; - long v; - if ((as = counterCells) != null && (n = as.length) > 0) { - if ((a = as[(n - 1) & h]) == null) { - if (cellsBusy == 0) { // Try to attach new Cell - CounterCell r = new CounterCell(x); // Optimistic create - if (cellsBusy == 0 && - Unsafe.getUnsafe().compareAndSwapInt(this, CELLSBUSY, 0, 1)) { - boolean created = false; - try { // Recheck under lock - CounterCell[] rs; - int m, j; - if ((rs = counterCells) != null && - (m = rs.length) > 0 && - rs[j = (m - 1) & h] == null) { - rs[j] = r; - created = true; - } - } finally { - cellsBusy = 0; - } - if (created) - break; - continue; // Slot is now non-empty - } - } - collide = false; - } else if (!wasUncontended) // CAS already known to fail - wasUncontended = true; // Continue after rehash - else if (Unsafe.cas(a, CELLVALUE, v = a.value, v + x)) - break; - else if (counterCells != as || n >= NCPU) - collide = false; // At max size or stale - else if (!collide) - collide = true; - else if (cellsBusy == 0 && - Unsafe.getUnsafe().compareAndSwapInt(this, CELLSBUSY, 0, 1)) { - try { - if (counterCells == as) {// Expand table unless stale - CounterCell[] rs = new CounterCell[n << 1]; - System.arraycopy(as, 0, rs, 0, n); - counterCells = rs; - } - } finally { - cellsBusy = 0; - } - collide = false; - continue; // Retry with expanded table - } - h = advanceProbe(h); - } else if (cellsBusy == 0 && counterCells == as && - Unsafe.getUnsafe().compareAndSwapInt(this, CELLSBUSY, 0, 1)) { - boolean init = false; - try { // Initialize table - if (counterCells == as) { - CounterCell[] rs = new CounterCell[2]; - rs[h & 1] = new CounterCell(x); - counterCells = rs; - init = true; - } - } finally { - cellsBusy = 0; - } - if (init) - break; - } else if (Unsafe.cas(this, BASECOUNT, v = baseCount, v + x)) - break; // Fall back on using base - } - } - - private Traverser getTraverser(Node[] tab) { - Traverser traverser = tlTraverser.get(); - int len = tab == null ? 0 : tab.length; - traverser.of(tab, len, len); - return traverser; - } - - /** - * Initializes table, using the size recorded in sizeCtl. - */ - private Node[] initTable() { - Node[] tab; - int sc; - while ((tab = table) == null || tab.length == 0) { - if ((sc = sizeCtl) < 0) - Os.pause(); // lost initialization race; just spin - else if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc, -1)) { - try { - if ((tab = table) == null || tab.length == 0) { - int n = (sc > 0) ? sc : DEFAULT_CAPACITY; - @SuppressWarnings("unchecked") - Node[] nt = (Node[]) new Node[n]; - table = tab = nt; - sc = n - (n >>> 2); - } - } finally { - sizeCtl = sc; - } - break; - } - } - return tab; - } - - private boolean keyEquals(final CharSequence lhs, final CharSequence rhs) { - return keyEquals(lhs, rhs, ics); - } - /* ---------------- Counter support -------------- */ - - private int keyHashCode(final CharSequence key) { - return keyHashCode(key, ics); - } - - /** - * Moves and/or copies the nodes in each bin to new table. See - * above for explanation. - */ - private void transfer(Node[] tab, Node[] nextTab) { - int n = tab.length, stride; - if ((stride = (NCPU > 1) ? (n >>> 3) / NCPU : n) < MIN_TRANSFER_STRIDE) - stride = MIN_TRANSFER_STRIDE; // subdivide range - if (nextTab == null) { // initiating - try { - @SuppressWarnings("unchecked") - Node[] nt = (Node[]) new Node[n << 1]; - nextTab = nt; - } catch (Throwable ex) { // try to cope with OOME - sizeCtl = Integer.MAX_VALUE; - return; - } - nextTable = nextTab; - transferIndex = n; - } - int nextn = nextTab.length; - ForwardingNode fwd = new ForwardingNode<>(nextTab, ics); - boolean advance = true; - boolean finishing = false; // to ensure sweep before committing nextTab - for (int i = 0, bound = 0; ; ) { - Node f; - int fh; - while (advance) { - int nextIndex, nextBound; - if (--i >= bound || finishing) - advance = false; - else if ((nextIndex = transferIndex) <= 0) { - i = -1; - advance = false; - } else if (Unsafe.getUnsafe().compareAndSwapInt - (this, TRANSFERINDEX, nextIndex, - nextBound = (nextIndex > stride ? - nextIndex - stride : 0))) { - bound = nextBound; - i = nextIndex - 1; - advance = false; - } - } - if (i < 0 || i >= n || i + n >= nextn) { - int sc; - if (finishing) { - nextTable = null; - table = nextTab; - sizeCtl = (n << 1) - (n >>> 1); - return; - } - if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc = sizeCtl, sc - 1)) { - if ((sc - 2) != resizeStamp(n) << RESIZE_STAMP_SHIFT) - return; - finishing = advance = true; - i = n; // recheck before commit - } - } else if ((f = tabAt(tab, i)) == null) - advance = casTabAt(tab, i, fwd); - else if ((fh = f.hash) == MOVED) - advance = true; // already processed - else { - synchronized (f) { - if (tabAt(tab, i) == f) { - Node ln, hn; - if (fh >= 0) { - int runBit = fh & n; - Node lastRun = f; - for (Node p = f.next; p != null; p = p.next) { - int b = p.hash & n; - if (b != runBit) { - runBit = b; - lastRun = p; - } - } - if (runBit == 0) { - ln = lastRun; - hn = null; - } else { - hn = lastRun; - ln = null; - } - for (Node p = f; p != lastRun; p = p.next) { - int ph = p.hash; - CharSequence pk = p.key; - V pv = p.val; - if ((ph & n) == 0) - ln = new Node<>(ph, pk, pv, ln, ics); - else - hn = new Node<>(ph, pk, pv, hn, ics); - } - setTabAt(nextTab, i, ln); - setTabAt(nextTab, i + n, hn); - setTabAt(tab, i, fwd); - advance = true; - } else if (f instanceof TreeBin) { - TreeBin t = (TreeBin) f; - TreeNode lo = null, loTail = null; - TreeNode hi = null, hiTail = null; - int lc = 0, hc = 0; - for (Node e = t.first; e != null; e = e.next) { - int h = e.hash; - TreeNode p = new TreeNode<> - (h, e.key, e.val, null, null, ics); - if ((h & n) == 0) { - if ((p.prev = loTail) == null) - lo = p; - else - loTail.next = p; - loTail = p; - ++lc; - } else { - if ((p.prev = hiTail) == null) - hi = p; - else - hiTail.next = p; - hiTail = p; - ++hc; - } - } - ln = (lc <= UNTREEIFY_THRESHOLD) ? untreeify(lo) : - (hc != 0) ? new TreeBin<>(lo, ics) : t; - hn = (hc <= UNTREEIFY_THRESHOLD) ? untreeify(hi) : - (lc != 0) ? new TreeBin<>(hi, ics) : t; - setTabAt(nextTab, i, ln); - setTabAt(nextTab, i + n, hn); - setTabAt(tab, i, fwd); - advance = true; - } - } - } - } - } - } - - /** - * Replaces all linked nodes in bin at given index unless table is - * too small, in which case resizes instead. - */ - private void treeifyBin(Node[] tab, int index) { - Node b; - int n; - if (tab != null) { - if ((n = tab.length) < MIN_TREEIFY_CAPACITY) - tryPresize(n << 1); - else if ((b = tabAt(tab, index)) != null && b.hash >= 0) { - synchronized (b) { - if (tabAt(tab, index) == b) { - TreeNode hd = null, tl = null; - for (Node e = b; e != null; e = e.next) { - TreeNode p = - new TreeNode<>(e.hash, e.key, e.val, null, null, ics); - if ((p.prev = tl) == null) - hd = p; - else - tl.next = p; - tl = p; - } - setTabAt(tab, index, new TreeBin<>(hd, ics)); - } - } - } - } - } - - /* ---------------- Conversion from/to TreeBins -------------- */ - - /** - * Tries to presize table to accommodate the given number of elements. - * - * @param size number of elements (doesn't need to be perfectly accurate) - */ - private void tryPresize(int size) { - int c = (size >= (MAXIMUM_CAPACITY >>> 1)) ? MAXIMUM_CAPACITY : - tableSizeFor(size + (size >>> 1) + 1); - int sc; - while ((sc = sizeCtl) >= 0) { - Node[] tab = table; - int n; - if (tab == null || (n = tab.length) == 0) { - n = Math.max(sc, c); - if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc, -1)) { - try { - if (table == tab) { - @SuppressWarnings("unchecked") - Node[] nt = (Node[]) new Node[n]; - table = nt; - sc = n - (n >>> 2); - } - } finally { - sizeCtl = sc; - } - } - } else if (c <= sc || n >= MAXIMUM_CAPACITY) - break; - else if (tab == table) { - int rs = resizeStamp(n); - if (sc < 0) { - Node[] nt; - if ((sc >>> RESIZE_STAMP_SHIFT) != rs || sc == rs + 1 || - sc == rs + MAX_RESIZERS || (nt = nextTable) == null || - transferIndex <= 0) - break; - if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc, sc + 1)) - transfer(tab, nt); - } else if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc, - (rs << RESIZE_STAMP_SHIFT) + 2)) - transfer(tab, null); - } - } - } - - static int advanceProbe(int probe) { - probe ^= probe << 13; // xorshift - probe ^= probe >>> 17; - probe ^= probe << 5; - Unsafe.getUnsafe().putInt(Thread.currentThread(), PROBE, probe); - return probe; - } - - /* ---------------- TreeNodes -------------- */ - - static boolean casTabAt(Node[] tab, int i, - Node v) { - return Unsafe.getUnsafe().compareAndSwapObject(tab, ((long) i << ASHIFT) + ABASE, null, v); - } - - /* ---------------- TreeBins -------------- */ - - /** - * Returns x's Class if it is of the form "class C implements - * Comparable", else null. - */ - static Class comparableClassFor(Object x) { - if (x instanceof Comparable) { - Class c; - Type[] ts, as; - Type t; - ParameterizedType p; - if ((c = x.getClass()) == String.class) // bypass checks - return c; - if ((ts = c.getGenericInterfaces()) != null) { - for (int i = 0; i < ts.length; ++i) { - if (((t = ts[i]) instanceof ParameterizedType) && - ((p = (ParameterizedType) t).getRawType() == - Comparable.class) && - (as = p.getActualTypeArguments()) != null && - as.length == 1 && as[0] == c) // type arg is c - return c; - } - } - } - return null; - } - - /* ----------------Table Traversal -------------- */ - - /** - * Returns k.compareTo(x) if x matches kc (k's screened comparable - * class), else 0. - */ - @SuppressWarnings({"rawtypes", "unchecked"}) // for cast to Comparable - static int compareComparables(Class kc, Object k, Object x) { - return (x == null || x.getClass() != kc ? 0 : - ((Comparable) k).compareTo(x)); - } - - static int getProbe() { - return Unsafe.getUnsafe().getInt(Thread.currentThread(), PROBE); - } - - /** - * Initialize Thread fields for the current thread. Called only - * when Thread.threadLocalRandomProbe is zero, indicating that a - * thread local seed value needs to be generated. Note that even - * though the initialization is purely thread-local, we need to - * rely on (static) atomic generators to initialize the values. - */ - static void localInit() { - int p = probeGenerator.addAndGet(PROBE_INCREMENT); - int probe = (p == 0) ? 1 : p; // skip 0 - long seed = mix64(seeder.getAndAdd(SEEDER_INCREMENT)); - Thread t = Thread.currentThread(); - Unsafe.getUnsafe().putLong(t, SEED, seed); - Unsafe.getUnsafe().putInt(t, PROBE, probe); - } - - /** - * Returns the stamp bits for resizing a table of size n. - * Must be negative when shifted left by RESIZE_STAMP_SHIFT. - */ - static int resizeStamp(int n) { - return Integer.numberOfLeadingZeros(n) | (1 << (RESIZE_STAMP_BITS - 1)); - } - - static void setTabAt(Node[] tab, int i, Node v) { - Unsafe.getUnsafe().putObjectVolatile(tab, ((long) i << ASHIFT) + ABASE, v); - } - - /** - * Spreads (XORs) higher bits of hash to lower and also forces top - * bit to 0. Because the table uses power-of-two masking, sets of - * hashes that vary only in bits above the current mask will - * always collide. (Among known examples are sets of Float keys - * holding consecutive whole numbers in small tables.) So we - * apply a transform that spreads the impact of higher bits - * downward. There is a tradeoff between speed, utility, and - * quality of bit-spreading. Because many common sets of hashes - * are already reasonably distributed (so don't benefit from - * spreading), and because we use trees to handle large sets of - * collisions in bins, we just XOR some shifted bits in the - * cheapest possible way to reduce systematic lossage, as well as - * to incorporate impact of the highest bits that would otherwise - * never be used in index calculations because of table bounds. - */ - static int spread(int h) { - return (h ^ (h >>> 16)) & HASH_BITS; - } - - @SuppressWarnings("unchecked") - static Node tabAt(Node[] tab, int i) { - return (Node) Unsafe.getUnsafe().getObjectVolatile(tab, ((long) i << ASHIFT) + ABASE); - } - - /* ----------------Views -------------- */ - - /** - * Returns a list on non-TreeNodes replacing those in given list. - */ - static Node untreeify(Node b) { - Node hd = null, tl = null; - for (Node q = b; q != null; q = q.next) { - Node p = new Node<>(q.hash, q.key, q.val, null, q.ics); - if (tl == null) - hd = p; - else - tl.next = p; - tl = p; - } - return hd; - } - - /** - * Helps transfer if a resize is in progress. - */ - final Node[] helpTransfer(Node[] tab, Node f) { - Node[] nextTab; - int sc; - if (tab != null && (f instanceof ForwardingNode) && - (nextTab = ((ForwardingNode) f).nextTable) != null) { - int rs = resizeStamp(tab.length); - while (nextTab == nextTable && table == tab && - (sc = sizeCtl) < 0) { - if ((sc >>> RESIZE_STAMP_SHIFT) != rs || sc == rs + 1 || - sc == rs + MAX_RESIZERS || transferIndex <= 0) - break; - if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc, sc + 1)) { - transfer(tab, nextTab); - break; - } - } - return nextTab; - } - return table; - } - - /** - * Implementation for put and putIfAbsent - */ - final V putVal(CharSequence key, V value, boolean onlyIfAbsent) { - if (key == null || value == null) throw new NullPointerException(); - int hash = spread(keyHashCode(key)); - int binCount = 0; - Node _new = null; - - for (Node[] tab = table; ; ) { - Node f; - int n, i, fh; - if (tab == null || (n = tab.length) == 0) - tab = initTable(); - else if ((f = tabAt(tab, i = (n - 1) & hash)) == null) { - if (_new == null) { - _new = new Node<>(hash, maybeCopyKey(key), value, null, ics); - } - if (casTabAt(tab, i, _new)) { - break; // no lock when adding to empty bin - } - } else if ((fh = f.hash) == MOVED) - tab = helpTransfer(tab, f); - else { - V oldVal = null; - synchronized (f) { - if (tabAt(tab, i) == f) { - if (fh >= 0) { - binCount = 1; - for (Node e = f; ; ++binCount) { - CharSequence ek; - if (e.hash == hash && - ((ek = e.key) == key || - (ek != null && keyEquals(key, ek)))) { - oldVal = e.val; - if (!onlyIfAbsent) - e.val = value; - break; - } - Node pred = e; - if ((e = e.next) == null) { - if (_new == null) { - pred.next = new Node<>(hash, maybeCopyKey(key), value, null, ics); - } else { - pred.next = _new; - } - break; - } - } - } else if (f instanceof TreeBin) { - Node p; - binCount = 2; - if ((p = ((TreeBin) f).putTreeVal(hash, maybeCopyKey(key), value)) != null) { - oldVal = p.val; - if (!onlyIfAbsent) - p.val = value; - } - } - } - } - if (binCount != 0) { - if (binCount >= TREEIFY_THRESHOLD) - treeifyBin(tab, i); - if (oldVal != null) - return oldVal; - break; - } - } - } - addCount(1L, binCount); - return null; - } - - /** - * Implementation for the four public remove/replace methods: - * Replaces node value with v, conditional upon match of cv if - * non-null. If resulting value is null, delete. - */ - final V replaceNode(CharSequence key, V value, V cv) { - int hash = spread(keyHashCode(key)); - for (Node[] tab = table; ; ) { - Node f; - int n, i, fh; - if (tab == null || (n = tab.length) == 0 || - (f = tabAt(tab, i = (n - 1) & hash)) == null) - break; - else if ((fh = f.hash) == MOVED) - tab = helpTransfer(tab, f); - else { - V oldVal = null; - boolean validated = false; - synchronized (f) { - if (tabAt(tab, i) == f) { - if (fh >= 0) { - validated = true; - for (Node e = f, pred = null; ; ) { - CharSequence ek; - if (e.hash == hash && - ((ek = e.key) == key || - (ek != null && keyEquals(key, ek)))) { - V ev = e.val; - if (cv == null || cv == ev || (cv.equals(ev))) { - oldVal = ev; - if (value != null) - e.val = value; - else if (pred != null) - pred.next = e.next; - else - setTabAt(tab, i, e.next); - } - break; - } - pred = e; - if ((e = e.next) == null) - break; - } - } else if (f instanceof TreeBin) { - validated = true; - TreeBin t = (TreeBin) f; - TreeNode r, p; - if ((r = t.root) != null && - (p = r.findTreeNode(hash, key, null)) != null) { - V pv = p.val; - if (cv == null || cv == pv || cv.equals(pv)) { - oldVal = pv; - if (value != null) - p.val = value; - else if (t.removeTreeNode(p)) - setTabAt(tab, i, untreeify(t.first)); - } - } - } - } - } - if (validated) { - if (oldVal != null) { - if (value == null) - addCount(-1L, -1); - return oldVal; - } - break; - } - } - } - return null; - } - - final long sumCount() { - CounterCell[] as = counterCells; - CounterCell a; - long sum = baseCount; - if (as != null) { - for (int i = 0; i < as.length; ++i) { - if ((a = as[i]) != null) - sum += a.value; - } - } - return sum; - } - - /** - * Base of key, value, and entry Iterators. Adds fields to - * Traverser to support iterator.remove. - */ - static class BaseIterator extends Traverser { - Node lastReturned; - ConcurrentHashMap map; - - public final boolean hasNext() { - return next != null; - } - - public final void remove() { - Node p; - if ((p = lastReturned) == null) - throw new IllegalStateException(); - lastReturned = null; - map.replaceNode(p.key, null, null); - } - - void of(ConcurrentHashMap map) { - Node[] tab = map.table; - int l = tab == null ? 0 : tab.length; - super.of(tab, l, l); - this.map = map; - advance(); - } - } - - /** - * Base class for views. - */ - abstract static class CollectionView - implements Collection, java.io.Serializable { - private static final String oomeMsg = "Required array size too large"; - private static final long serialVersionUID = 7249069246763182397L; - final ConcurrentHashMap map; - - CollectionView(ConcurrentHashMap map) { - this.map = map; - } - - /** - * Removes all of the elements from this view, by removing all - * the mappings from the map backing this view. - */ - public final void clear() { - map.clear(); - } - - public abstract boolean contains(Object o); - - public final boolean containsAll(@NotNull Collection c) { - if (c != this) { - for (Object e : c) { - if (e == null || !contains(e)) - return false; - } - } - return true; - } - - /** - * Returns the map backing this view. - * - * @return the map backing this view - */ - public ConcurrentHashMap getMap() { - return map; - } - - public final boolean isEmpty() { - return map.isEmpty(); - } - - /** - * Returns an iterator over the elements in this collection. - *

    The returned iterator is - * weakly consistent. - * - * @return an iterator over the elements in this collection - */ - @NotNull - public abstract Iterator iterator(); - - public abstract boolean remove(Object o); - - public final boolean removeAll(@NotNull Collection c) { - boolean modified = false; - for (Iterator it = iterator(); it.hasNext(); ) { - if (c.contains(it.next())) { - it.remove(); - modified = true; - } - } - return modified; - } - - - // implementations below rely on concrete classes supplying these - // abstract methods - - public final boolean retainAll(@NotNull Collection c) { - boolean modified = false; - for (Iterator it = iterator(); it.hasNext(); ) { - if (!c.contains(it.next())) { - it.remove(); - modified = true; - } - } - return modified; - } - - public final int size() { - return map.size(); - } - - @NotNull - public final Object[] toArray() { - long sz = map.mappingCount(); - if (sz > MAX_ARRAY_SIZE) - throw new OutOfMemoryError(oomeMsg); - int n = (int) sz; - Object[] r = new Object[n]; - int i = 0; - for (E e : this) { - if (i == n) { - if (n >= MAX_ARRAY_SIZE) - throw new OutOfMemoryError(oomeMsg); - if (n >= MAX_ARRAY_SIZE - (MAX_ARRAY_SIZE >>> 1) - 1) - n = MAX_ARRAY_SIZE; - else - n += (n >>> 1) + 1; - r = Arrays.copyOf(r, n); - } - r[i++] = e; - } - return (i == n) ? r : Arrays.copyOf(r, i); - } - - @NotNull - @SuppressWarnings("unchecked") - public final T[] toArray(@NotNull T[] a) { - long sz = map.mappingCount(); - if (sz > MAX_ARRAY_SIZE) - throw new OutOfMemoryError(oomeMsg); - int m = (int) sz; - T[] r = (a.length >= m) ? a : - (T[]) java.lang.reflect.Array - .newInstance(a.getClass().getComponentType(), m); - int n = r.length; - int i = 0; - for (E e : this) { - if (i == n) { - if (n >= MAX_ARRAY_SIZE) - throw new OutOfMemoryError(oomeMsg); - if (n >= MAX_ARRAY_SIZE - (MAX_ARRAY_SIZE >>> 1) - 1) - n = MAX_ARRAY_SIZE; - else - n += (n >>> 1) + 1; - r = Arrays.copyOf(r, n); - } - r[i++] = (T) e; - } - if (a == r && i < n) { - r[i] = null; // null-terminate - return r; - } - return (i == n) ? r : Arrays.copyOf(r, i); - } - - /** - * Returns a string representation of this collection. - * The string representation consists of the string representations - * of the collection's elements in the order they are returned by - * its iterator, enclosed in square brackets ({@code "[]"}). - * Adjacent elements are separated by the characters {@code ", "} - * (comma and space). Elements are converted to strings as by - * {@link String#valueOf(Object)}. - * - * @return a string representation of this collection - */ - public final String toString() { - StringBuilder sb = new StringBuilder(); - sb.append('['); - Iterator it = iterator(); - if (it.hasNext()) { - for (; ; ) { - Object e = it.next(); - sb.append(e == this ? "(this Collection)" : e); - if (!it.hasNext()) - break; - sb.append(',').append(' '); - } - } - return sb.append(']').toString(); - } - - } - - /** - * A padded cell for distributing counts. Adapted from LongAdder - * and Striped64. See their internal docs for explanation. - */ - static final class CounterCell { - final long value; - - CounterCell(long x) { - value = x; - } - } - - static final class EntryIterator extends BaseIterator - implements Iterator> { - - public Map.Entry next() { - Node p; - if ((p = next) == null) - throw new NoSuchElementException(); - CharSequence k = p.key; - V v = p.val; - lastReturned = p; - advance(); - return new MapEntry<>(k, v, map); - } - } - - /** - * A view of a ConcurrentHashMap as a {@link Set} of (key, value) - * entries. This class cannot be directly instantiated. See - * {@link #entrySet()}. - */ - static final class EntrySetView extends CollectionView> - implements Set>, java.io.Serializable { - private static final long serialVersionUID = 2249069246763182397L; - - private final ThreadLocal> tlEntryIterator = ThreadLocal.withInitial(EntryIterator::new); - - EntrySetView(ConcurrentHashMap map) { - super(map); - } - - public boolean add(Entry e) { - return map.putVal(e.getKey(), e.getValue(), false) == null; - } - - public boolean addAll(@NotNull Collection> c) { - boolean added = false; - for (Entry e : c) { - if (add(e)) - added = true; - } - return added; - } - - public boolean contains(Object o) { - Object k, v, r; - Map.Entry e; - return ((o instanceof Map.Entry) && - (k = (e = (Map.Entry) o).getKey()) != null && - (r = map.get(k)) != null && - (v = e.getValue()) != null && - (v == r || v.equals(r))); - } - - public boolean equals(Object o) { - Set c; - return ((o instanceof Set) && - ((c = (Set) o) == this || - (containsAll(c) && c.containsAll(this)))); - } - - public int hashCode() { - int h = 0; - Node[] t = map.table; - if (t != null) { - Traverser it = map.getTraverser(t); - for (Node p; (p = it.advance()) != null; ) { - h += p.hashCode(); - } - } - return h; - } - - /** - * @return an iterator over the entries of the backing map - */ - @NotNull - public Iterator> iterator() { - EntryIterator it = tlEntryIterator.get(); - it.of(map); - return it; - } - - public boolean remove(Object o) { - Object k, v; - Map.Entry e; - return ((o instanceof Map.Entry) && - (k = (e = (Map.Entry) o).getKey()) != null && - (v = e.getValue()) != null && - map.remove(k, v)); - } - } - - /** - * A node inserted at head of bins during transfer operations. - */ - static final class ForwardingNode extends Node { - final Node[] nextTable; - - ForwardingNode(Node[] tab, boolean ics) { - super(MOVED, null, null, null, ics); - this.nextTable = tab; - } - - Node find(int h, CharSequence k) { - // loop to avoid arbitrarily deep recursion on forwarding nodes - outer: - for (Node[] tab = nextTable; ; ) { - Node e; - int n; - if (k == null || tab == null || (n = tab.length) == 0 || - (e = tabAt(tab, (n - 1) & h)) == null) - return null; - for (; ; ) { - int eh; - CharSequence ek; - if ((eh = e.hash) == h && - ((ek = e.key) == k || (ek != null && keyEquals(k, ek, ics)))) - return e; - if (eh < 0) { - if (e instanceof ForwardingNode) { - tab = ((ForwardingNode) e).nextTable; - continue outer; - } else - return e.find(h, k); - } - if ((e = e.next) == null) - return null; - } - } - } - } - - static final class KeyIterator extends BaseIterator - implements Iterator { - - public CharSequence next() { - Node p; - if ((p = next) == null) - throw new NoSuchElementException(); - CharSequence k = p.key; - lastReturned = p; - advance(); - return k; - } - } - - /** - * A view of a ConcurrentHashMap as a {@link Set} of keys, in - * which additions may optionally be enabled by mapping to a - * common value. This class cannot be directly instantiated. - * See {@link #keySet() keySet()}, - * {@link #keySet(Object) keySet(V)}, - * {@link #newKeySet() newKeySet()}, - * {@link #newKeySet(int) newKeySet(int)}. - * - * @since 1.8 - */ - public static class KeySetView extends CollectionView - implements Set, java.io.Serializable { - private static final long serialVersionUID = 7249069246763182397L; - private final ThreadLocal> tlKeyIterator = ThreadLocal.withInitial(KeyIterator::new); - private final V value; - - KeySetView(ConcurrentHashMap map, V value) { // non-public - super(map); - this.value = value; - } - - /** - * Adds the specified key to this set view by mapping the key to - * the default mapped value in the backing map, if defined. - * - * @param e key to be added - * @return {@code true} if this set changed as a result of the call - * @throws NullPointerException if the specified key is null - * @throws UnsupportedOperationException if no default mapped value - * for additions was provided - */ - public boolean add(CharSequence e) { - V v; - if ((v = value) == null) - throw new UnsupportedOperationException(); - return map.putVal(e, v, true) == null; - } - - /** - * Adds all of the elements in the specified collection to this set, - * as if by calling {@link #add} on each one. - * - * @param c the elements to be inserted into this set - * @return {@code true} if this set changed as a result of the call - * @throws NullPointerException if the collection or any of its - * elements are {@code null} - * @throws UnsupportedOperationException if no default mapped value - * for additions was provided - */ - public boolean addAll(@NotNull Collection c) { - boolean added = false; - V v; - if ((v = value) == null) - throw new UnsupportedOperationException(); - for (CharSequence e : c) { - if (map.putVal(e, v, true) == null) - added = true; - } - return added; - } - - /** - * {@inheritDoc} - * - * @throws NullPointerException if the specified key is null - */ - public boolean contains(Object o) { - return map.containsKey(o); - } - - public boolean equals(Object o) { - Set c; - return ((o instanceof Set) && - ((c = (Set) o) == this || - (containsAll(c) && c.containsAll(this)))); - } - - /** - * Returns the default mapped value for additions, - * or {@code null} if additions are not supported. - * - * @return the default mapped value for additions, or {@code null} - * if not supported - */ - public V getMappedValue() { - return value; - } - - public int hashCode() { - int h = 0; - for (CharSequence e : this) - h += e.hashCode(); - return h; - } - - /** - * @return an iterator over the keys of the backing map - */ - @NotNull - public Iterator iterator() { - KeyIterator it = tlKeyIterator.get(); - it.of(map); - return it; - } - - /** - * Removes the key from this map view, by removing the key (and its - * corresponding value) from the backing map. This method does - * nothing if the key is not in the map. - * - * @param o the key to be removed from the backing map - * @return {@code true} if the backing map contained the specified key - * @throws NullPointerException if the specified key is null - */ - public boolean remove(Object o) { - return map.remove(o) != null; - } - } - - /** - * Exported Entry for EntryIterator - */ - static final class MapEntry implements Map.Entry { - final CharSequence key; // non-null - final ConcurrentHashMap map; - V val; // non-null - - MapEntry(CharSequence key, V val, ConcurrentHashMap map) { - this.key = key; - this.val = val; - this.map = map; - } - - public boolean equals(Object o) { - Object k, v; - Map.Entry e; - return ((o instanceof Map.Entry) && - (k = (e = (Map.Entry) o).getKey()) != null && - (v = e.getValue()) != null && - (k == key || map.keyEquals((CharSequence) k, key)) && - (v == val || v.equals(val))); - } - - public CharSequence getKey() { - return key; - } - - public V getValue() { - return val; - } - - public int hashCode() { - return map.keyHashCode(key) ^ val.hashCode(); - } - - /** - * Sets our entry's value and writes through to the map. The - * value to return is somewhat arbitrary here. Since we do not - * necessarily track asynchronous changes, the most recent - * "previous" value could be different from what we return (or - * could even have been removed, in which case the put will - * re-establish). We do not and cannot guarantee more. - */ - public V setValue(V value) { - if (value == null) throw new NullPointerException(); - V v = val; - val = value; - map.put(key, value); - return v; - } - - public String toString() { - return key + "=" + val; - } - } - - /** - * Key-value entry. This class is never exported out as a - * user-mutable Map.Entry (i.e., one supporting setValue; see - * MapEntry below), but can be used for read-only traversals used - * in bulk tasks. Subclasses of Node with a negative hash field - * are special, and contain null keys and values (but are never - * exported). Otherwise, keys and vals are never null. - */ - static class Node implements Map.Entry { - final int hash; - final boolean ics; - final CharSequence key; - volatile Node next; - volatile V val; - - Node(int hash, CharSequence key, V val, Node next, boolean ics) { - this.hash = hash; - this.key = key; - this.val = val; - this.next = next; - this.ics = ics; - } - - public final boolean equals(Object o) { - Object k, v, u; - Map.Entry e; - return ((o instanceof Map.Entry) && - (k = (e = (Map.Entry) o).getKey()) != null && - (v = e.getValue()) != null && - (k == key || keyEquals((CharSequence) k, key, ics)) && - (v == (u = val) || v.equals(u))); - } - - public final CharSequence getKey() { - return key; - } - - public final V getValue() { - return val; - } - - public final int hashCode() { - return keyHashCode(key, ics) ^ val.hashCode(); - } - - public final V setValue(V value) { - throw new UnsupportedOperationException(); - } - - public final String toString() { - return key + "=" + val; - } - - /** - * Virtualized support for map.get(); overridden in subclasses. - */ - Node find(int h, CharSequence k) { - Node e = this; - if (k != null) { - do { - CharSequence ek; - if (e.hash == h && - ((ek = e.key) == k || (ek != null && keyEquals(k, ek, ics)))) - return e; - } while ((e = e.next) != null); - } - return null; - } - } - - /** - * A place-holder node used in computeIfAbsent and compute - */ - static final class ReservationNode extends Node { - ReservationNode(boolean ics) { - super(RESERVED, null, null, null, ics); - } - - Node find(int h, Object k) { - return null; - } - } - - /** - * Stripped-down version of helper class used in previous version, - * declared for the sake of serialization compatibility - */ - static class Segment extends ReentrantLock implements Serializable { - private static final long serialVersionUID = 2249069246763182397L; - final float loadFactor; - - Segment() { - this.loadFactor = ConcurrentHashMap.LOAD_FACTOR; - } - } - - /** - * Records the table, its length, and current traversal index for a - * traverser that must process a region of a forwarded table before - * proceeding with current table. - */ - static final class TableStack { - int index; - int length; - TableStack next; - Node[] tab; - } - - /** - * Encapsulates traversal for methods such as containsValue; also - * serves as a base class for other iterators and spliterators. - *

    - * Method advance visits once each still-valid node that was - * reachable upon iterator construction. It might miss some that - * were added to a bin after the bin was visited, which is OK wrt - * consistency guarantees. Maintaining this property in the face - * of possible ongoing resizes requires a fair amount of - * bookkeeping state that is difficult to optimize away amidst - * volatile accesses. Even so, traversal maintains reasonable - * throughput. - *

    - * Normally, iteration proceeds bin-by-bin traversing lists. - * However, if the table has been resized, then all future steps - * must traverse both the bin at the current index as well as at - * (index + baseSize); and so on for further resizings. To - * paranoically cope with potential sharing by users of iterators - * across threads, iteration terminates if a bounds checks fails - * for a table read. - */ - static class Traverser { - int baseIndex; // current index of initial table - int baseLimit; // index bound for initial table - int baseSize; // initial table size - int index; // index of bin to use next - Node next; // the next entry to use - TableStack stack, spare; // to save/restore on ForwardingNodes - Node[] tab; // current table; updated if resized - - /** - * Saves traversal state upon encountering a forwarding node. - */ - private void pushState(Node[] t, int i, int n) { - TableStack s = spare; // reuse if possible - if (s != null) - spare = s.next; - else - s = new TableStack<>(); - s.tab = t; - s.length = n; - s.index = i; - s.next = stack; - stack = s; - } - - /** - * Possibly pops traversal state. - * - * @param n length of current table - */ - private void recoverState(int n) { - TableStack s; - int len; - while ((s = stack) != null && (index += (len = s.length)) >= n) { - n = len; - index = s.index; - tab = s.tab; - s.tab = null; - TableStack next = s.next; - s.next = spare; // save for reuse - stack = next; - spare = s; - } - if (s == null && (index += baseSize) >= n) - index = ++baseIndex; - } - - /** - * Advances if possible, returning next valid node, or null if none. - */ - final Node advance() { - Node e; - if ((e = next) != null) - e = e.next; - for (; ; ) { - Node[] t; - int i, n; // must use locals in checks - if (e != null) - return next = e; - if (baseIndex >= baseLimit || (t = tab) == null || - (n = t.length) <= (i = index) || i < 0) - return next = null; - if ((e = tabAt(t, i)) != null && e.hash < 0) { - if (e instanceof ForwardingNode) { - tab = ((ForwardingNode) e).nextTable; - e = null; - pushState(t, i, n); - continue; - } else if (e instanceof TreeBin) - e = ((TreeBin) e).first; - else - e = null; - } - if (stack != null) - recoverState(n); - else if ((index = i + baseSize) >= n) - index = ++baseIndex; // visit upper slots if present - } - } - - void of(Node[] tab, int size, int limit) { - this.tab = tab; - this.baseSize = size; - this.baseIndex = this.index = 0; - this.baseLimit = limit; - this.next = null; - } - } - - /** - * TreeNodes used at the heads of bins. TreeBins do not hold user - * keys or values, but instead point to list of TreeNodes and - * their root. They also maintain a parasitic read-write lock - * forcing writers (who hold bin lock) to wait for readers (who do - * not) to complete before tree restructuring operations. - */ - static final class TreeBin extends Node { - static final int READER = 4; // increment value for setting read lock - static final int WAITER = 2; // set when waiting for write lock - // values for lockState - static final int WRITER = 1; // set while holding write lock - private static final long LOCKSTATE; - private static final sun.misc.Unsafe U; - volatile TreeNode first; - volatile int lockState; - TreeNode root; - volatile Thread waiter; - - /** - * Creates bin with initial set of nodes headed by b. - */ - TreeBin(TreeNode b, boolean ics) { - super(TREEBIN, null, null, null, ics); - this.first = b; - TreeNode r = null; - for (TreeNode x = b, next; x != null; x = next) { - next = (TreeNode) x.next; - x.left = x.right = null; - if (r == null) { - x.parent = null; - x.red = false; - r = x; - } else { - CharSequence k = x.key; - int h = x.hash; - Class kc = null; - for (TreeNode p = r; ; ) { - int dir, ph; - CharSequence pk = p.key; - if ((ph = p.hash) > h) - dir = -1; - else if (ph < h) - dir = 1; - else if ((kc == null && - (kc = comparableClassFor(k)) == null) || - (dir = compareComparables(kc, k, pk)) == 0) - dir = tieBreakOrder(k, pk); - TreeNode xp = p; - if ((p = (dir <= 0) ? p.left : p.right) == null) { - x.parent = xp; - if (dir <= 0) - xp.left = x; - else - xp.right = x; - r = balanceInsertion(r, x); - break; - } - } - } - } - this.root = r; - assert checkInvariants(root); - } - - /** - * Possibly blocks awaiting root lock. - */ - private void contendedLock() { - boolean waiting = false; - for (int s; ; ) { - if (((s = lockState) & ~WAITER) == 0) { - if (U.compareAndSwapInt(this, LOCKSTATE, s, WRITER)) { - if (waiting) - waiter = null; - return; - } - } else if ((s & WAITER) == 0) { - if (U.compareAndSwapInt(this, LOCKSTATE, s, s | WAITER)) { - waiting = true; - waiter = Thread.currentThread(); - } - } else if (waiting) - LockSupport.park(this); - } - } - - /** - * Acquires write lock for tree restructuring. - */ - private void lockRoot() { - if (!U.compareAndSwapInt(this, LOCKSTATE, 0, WRITER)) - contendedLock(); // offload to separate method - } - - /** - * Releases write lock for tree restructuring. - */ - private void unlockRoot() { - lockState = 0; - } - - static TreeNode balanceDeletion(TreeNode root, - TreeNode x) { - for (TreeNode xp, xpl, xpr; ; ) { - if (x == null || x == root) - return root; - else if ((xp = x.parent) == null) { - x.red = false; - return x; - } else if (x.red) { - x.red = false; - return root; - } else if ((xpl = xp.left) == x) { - if ((xpr = xp.right) != null && xpr.red) { - xpr.red = false; - xp.red = true; - root = rotateLeft(root, xp); - xpr = (xp = x.parent) == null ? null : xp.right; - } - if (xpr == null) - x = xp; - else { - TreeNode sl = xpr.left, sr = xpr.right; - if ((sr == null || !sr.red) && - (sl == null || !sl.red)) { - xpr.red = true; - x = xp; - } else { - if (sr == null || !sr.red) { - sl.red = false; - xpr.red = true; - root = rotateRight(root, xpr); - xpr = (xp = x.parent) == null ? - null : xp.right; - } - if (xpr != null) { - xpr.red = xp.red; - if ((sr = xpr.right) != null) - sr.red = false; - } - if (xp != null) { - xp.red = false; - root = rotateLeft(root, xp); - } - x = root; - } - } - } else { // symmetric - if (xpl != null && xpl.red) { - xpl.red = false; - xp.red = true; - root = rotateRight(root, xp); - xpl = (xp = x.parent) == null ? null : xp.left; - } - if (xpl == null) - x = xp; - else { - TreeNode sl = xpl.left, sr = xpl.right; - if ((sl == null || !sl.red) && - (sr == null || !sr.red)) { - xpl.red = true; - x = xp; - } else { - if (sl == null || !sl.red) { - sr.red = false; - xpl.red = true; - root = rotateLeft(root, xpl); - xpl = (xp = x.parent) == null ? - null : xp.left; - } - if (xpl != null) { - xpl.red = xp.red; - if ((sl = xpl.left) != null) - sl.red = false; - } - if (xp != null) { - xp.red = false; - root = rotateRight(root, xp); - } - x = root; - } - } - } - } - } - - static TreeNode balanceInsertion(TreeNode root, - TreeNode x) { - x.red = true; - for (TreeNode xp, xpp, xppl, xppr; ; ) { - if ((xp = x.parent) == null) { - x.red = false; - return x; - } else if (!xp.red || (xpp = xp.parent) == null) - return root; - if (xp == (xppl = xpp.left)) { - if ((xppr = xpp.right) != null && xppr.red) { - xppr.red = false; - xp.red = false; - xpp.red = true; - x = xpp; - } else { - if (x == xp.right) { - root = rotateLeft(root, x = xp); - xpp = (xp = x.parent) == null ? null : xp.parent; - } - if (xp != null) { - xp.red = false; - if (xpp != null) { - xpp.red = true; - root = rotateRight(root, xpp); - } - } - } - } else { - if (xppl != null && xppl.red) { - xppl.red = false; - xp.red = false; - xpp.red = true; - x = xpp; - } else { - if (x == xp.left) { - root = rotateRight(root, x = xp); - xpp = (xp = x.parent) == null ? null : xp.parent; - } - if (xp != null) { - xp.red = false; - if (xpp != null) { - xpp.red = true; - root = rotateLeft(root, xpp); - } - } - } - } - } - } - - /** - * Recursive invariant check - */ - @SuppressWarnings("SimplifiableIfStatement") - static boolean checkInvariants(TreeNode t) { - TreeNode tp = t.parent, tl = t.left, tr = t.right, - tb = t.prev, tn = (TreeNode) t.next; - if (tb != null && tb.next != t) - return false; - if (tn != null && tn.prev != t) - return false; - if (tp != null && t != tp.left && t != tp.right) - return false; - if (tl != null && (tl.parent != t || tl.hash > t.hash)) - return false; - if (tr != null && (tr.parent != t || tr.hash < t.hash)) - return false; - if (t.red && tl != null && tl.red && tr != null && tr.red) - return false; - if (tl != null && !checkInvariants(tl)) - return false; - return !(tr != null && !checkInvariants(tr)); - } - - static TreeNode rotateLeft(TreeNode root, TreeNode p) { - TreeNode r, pp, rl; - if (p != null && (r = p.right) != null) { - if ((rl = p.right = r.left) != null) - rl.parent = p; - if ((pp = r.parent = p.parent) == null) - (root = r).red = false; - else if (pp.left == p) - pp.left = r; - else - pp.right = r; - r.left = p; - p.parent = r; - } - return root; - } - - static TreeNode rotateRight(TreeNode root, TreeNode p) { - TreeNode l, pp, lr; - if (p != null && (l = p.left) != null) { - if ((lr = p.left = l.right) != null) - lr.parent = p; - if ((pp = l.parent = p.parent) == null) - (root = l).red = false; - else if (pp.right == p) - pp.right = l; - else - pp.left = l; - l.right = p; - p.parent = l; - } - return root; - } - - /** - * Tie-breaking utility for ordering insertions when equal - * hashCodes and non-comparable. We don't require a total - * order, just a consistent insertion rule to maintain - * equivalence across rebalancings. Tie-breaking further than - * necessary simplifies testing a bit. - */ - static int tieBreakOrder(Object a, Object b) { - int d; - if (a == null || b == null || - (d = a.getClass().getName(). - compareTo(b.getClass().getName())) == 0) - d = (System.identityHashCode(a) <= System.identityHashCode(b) ? - -1 : 1); - return d; - } - - /** - * Returns matching node or null if none. Tries to search - * using tree comparisons from root, but continues linear - * search when lock not available. - */ - Node find(int h, CharSequence k) { - if (k != null) { - for (Node e = first; e != null; ) { - int s; - CharSequence ek; - if (((s = lockState) & (WAITER | WRITER)) != 0) { - if (e.hash == h && - ((ek = e.key) == k || (ek != null && keyEquals(k, ek, ics)))) - return e; - e = e.next; - } else if (U.compareAndSwapInt(this, LOCKSTATE, s, - s + READER)) { - TreeNode r, p; - try { - p = ((r = root) == null ? null : - r.findTreeNode(h, k, null)); - } finally { - Thread w; - if (U.getAndAddInt(this, LOCKSTATE, -READER) == - (READER | WAITER) && (w = waiter) != null) - LockSupport.unpark(w); - } - return p; - } - } - } - return null; - } - - /** - * Finds or adds a node. - * - * @return null if added - */ - TreeNode putTreeVal(int h, CharSequence k, V v) { - Class kc = null; - boolean searched = false; - for (TreeNode p = root; ; ) { - int dir, ph; - CharSequence pk; - if (p == null) { - first = root = new TreeNode<>(h, k, v, null, null, ics); - break; - } else if ((ph = p.hash) > h) - dir = -1; - else if (ph < h) - dir = 1; - else if ((pk = p.key) == k || (pk != null && keyEquals(k, pk, ics))) - return p; - else if ((kc == null && - (kc = comparableClassFor(k)) == null) || - (dir = compareComparables(kc, k, pk)) == 0) { - if (!searched) { - TreeNode q, ch; - searched = true; - if (((ch = p.left) != null && - (q = ch.findTreeNode(h, k, kc)) != null) || - ((ch = p.right) != null && - (q = ch.findTreeNode(h, k, kc)) != null)) - return q; - } - dir = tieBreakOrder(k, pk); - } - - TreeNode xp = p; - if ((p = (dir <= 0) ? p.left : p.right) == null) { - TreeNode x, f = first; - first = x = new TreeNode<>(h, k, v, f, xp, ics); - if (f != null) - f.prev = x; - if (dir <= 0) - xp.left = x; - else - xp.right = x; - if (!xp.red) - x.red = true; - else { - lockRoot(); - try { - root = balanceInsertion(root, x); - } finally { - unlockRoot(); - } - } - break; - } - } - assert checkInvariants(root); - return null; - } - - /** - * Removes the given node, that must be present before this - * call. This is messier than typical red-black deletion code - * because we cannot swap the contents of an interior node - * with a leaf successor that is pinned by "next" pointers - * that are accessible independently of lock. So instead we - * swap the tree linkages. - * - * @return true if now too small, so should be untreeified - */ - boolean removeTreeNode(TreeNode p) { - TreeNode next = (TreeNode) p.next; - TreeNode pred = p.prev; // unlink traversal pointers - TreeNode r, rl; - if (pred == null) - first = next; - else - pred.next = next; - if (next != null) - next.prev = pred; - if (first == null) { - root = null; - return true; - } - if ((r = root) == null || r.right == null || // too small - (rl = r.left) == null || rl.left == null) - return true; - lockRoot(); - try { - TreeNode replacement; - TreeNode pl = p.left; - TreeNode pr = p.right; - if (pl != null && pr != null) { - TreeNode s = pr, sl; - while ((sl = s.left) != null) // find successor - s = sl; - boolean c = s.red; - s.red = p.red; - p.red = c; // swap colors - TreeNode sr = s.right; - TreeNode pp = p.parent; - if (s == pr) { // p was s's direct parent - p.parent = s; - s.right = p; - } else { - TreeNode sp = s.parent; - if ((p.parent = sp) != null) { - if (s == sp.left) - sp.left = p; - else - sp.right = p; - } - s.right = pr; - pr.parent = s; - } - p.left = null; - if ((p.right = sr) != null) - sr.parent = p; - s.left = pl; - pl.parent = s; - if ((s.parent = pp) == null) - r = s; - else if (p == pp.left) - pp.left = s; - else - pp.right = s; - if (sr != null) - replacement = sr; - else - replacement = p; - } else if (pl != null) - replacement = pl; - else if (pr != null) - replacement = pr; - else - replacement = p; - if (replacement != p) { - TreeNode pp = replacement.parent = p.parent; - if (pp == null) - r = replacement; - else if (p == pp.left) - pp.left = replacement; - else - pp.right = replacement; - p.left = p.right = p.parent = null; - } - - root = (p.red) ? r : balanceDeletion(r, replacement); - - if (p == replacement) { // detach pointers - TreeNode pp; - if ((pp = p.parent) != null) { - if (p == pp.left) - pp.left = null; - else if (p == pp.right) - pp.right = null; - p.parent = null; - } - } - } finally { - unlockRoot(); - } - assert checkInvariants(root); - return false; - } - - static { - try { - U = Unsafe.getUnsafe(); - Class k = TreeBin.class; - LOCKSTATE = U.objectFieldOffset - (k.getDeclaredField("lockState")); - } catch (Exception e) { - throw new Error(e); - } - } - - - - - - - - - - - - - - - - /* ------------------------------------------------------------ */ - // Red-black tree methods, all adapted from CLR - } - - /** - * Nodes for use in TreeBins - */ - static final class TreeNode extends Node { - TreeNode left; - TreeNode parent; // red-black tree links - TreeNode prev; // needed to unlink next upon deletion - boolean red; - TreeNode right; - - TreeNode(int hash, CharSequence key, V val, Node next, - TreeNode parent, boolean ics) { - super(hash, key, val, next, ics); - this.parent = parent; - } - - Node find(int h, CharSequence k) { - return findTreeNode(h, k, null); - } - - /** - * Returns the TreeNode (or null if not found) for the given key - * starting at given root. - */ - TreeNode findTreeNode(int h, CharSequence k, Class kc) { - if (k != null) { - TreeNode p = this; - do { - int ph, dir; - CharSequence pk; - TreeNode q; - TreeNode pl = p.left, pr = p.right; - if ((ph = p.hash) > h) - p = pl; - else if (ph < h) - p = pr; - else if ((pk = p.key) == k || (pk != null && keyEquals(k, pk, ics))) - return p; - else if (pl == null) - p = pr; - else if (pr == null) - p = pl; - else if ((kc != null || - (kc = comparableClassFor(k)) != null) && - (dir = compareComparables(kc, k, pk)) != 0) - p = (dir < 0) ? pl : pr; - else if ((q = pr.findTreeNode(h, k, kc)) != null) - return q; - else - p = pl; - } while (p != null); - } - return null; - } - - - } - - static final class ValueIterator extends BaseIterator - implements Iterator { - public V next() { - Node p; - if ((p = next) == null) - throw new NoSuchElementException(); - V v = p.val; - lastReturned = p; - advance(); - return v; - } - } - - /** - * A view of a ConcurrentHashMap as a {@link Collection} of - * values, in which additions are disabled. This class cannot be - * directly instantiated. See {@link #values()}. - */ - static final class ValuesView extends CollectionView - implements Collection, java.io.Serializable { - private static final long serialVersionUID = 2249069246763182397L; - private final ThreadLocal> tlValueIterator = ThreadLocal.withInitial(ValueIterator::new); - - ValuesView(ConcurrentHashMap map) { - super(map); - } - - public boolean add(V e) { - throw new UnsupportedOperationException(); - } - - public boolean addAll(@NotNull Collection c) { - throw new UnsupportedOperationException(); - } - - public boolean contains(Object o) { - return map.containsValue(o); - } - - @NotNull - public Iterator iterator() { - ValueIterator it = tlValueIterator.get(); - it.of(map); - return it; - } - - public boolean remove(Object o) { - if (o != null) { - for (Iterator it = iterator(); it.hasNext(); ) { - if (o.equals(it.next())) { - it.remove(); - return true; - } - } - } - return false; - } - } - - static { - try { - Class tk = Thread.class; - SEED = Unsafe.getUnsafe().objectFieldOffset(tk.getDeclaredField("threadLocalRandomSeed")); - PROBE = Unsafe.getUnsafe().objectFieldOffset(tk.getDeclaredField("threadLocalRandomProbe")); - } catch (Exception e) { - throw new Error(e); - } - } - - static { - try { - Class k = ConcurrentHashMap.class; - SIZECTL = Unsafe.getUnsafe().objectFieldOffset - (k.getDeclaredField("sizeCtl")); - TRANSFERINDEX = Unsafe.getUnsafe().objectFieldOffset - (k.getDeclaredField("transferIndex")); - BASECOUNT = Unsafe.getUnsafe().objectFieldOffset - (k.getDeclaredField("baseCount")); - CELLSBUSY = Unsafe.getUnsafe().objectFieldOffset - (k.getDeclaredField("cellsBusy")); - Class ck = CounterCell.class; - CELLVALUE = Unsafe.getUnsafe().objectFieldOffset - (ck.getDeclaredField("value")); - Class ak = Node[].class; - ABASE = Unsafe.getUnsafe().arrayBaseOffset(ak); - int scale = Unsafe.getUnsafe().arrayIndexScale(ak); - if ((scale & (scale - 1)) != 0) - throw new Error("data type scale not a power of two"); - ASHIFT = 31 - Integer.numberOfLeadingZeros(scale); - } catch (Exception e) { - throw new Error(e); - } - } -} diff --git a/core/src/main/java/io/questdb/client/std/ConcurrentIntHashMap.java b/core/src/main/java/io/questdb/client/std/ConcurrentIntHashMap.java deleted file mode 100644 index a2f62ed..0000000 --- a/core/src/main/java/io/questdb/client/std/ConcurrentIntHashMap.java +++ /dev/null @@ -1,3612 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.std; - -/* - * ORACLE PROPRIETARY/CONFIDENTIAL. Use is subject to license terms. - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - */ - -/* - * - * - * - * - * - * Written by Doug Lea with assistance from members of JCP JSR-166 - * Expert Group and released to the public domain, as explained at - * http://creativecommons.org/publicdomain/zero/1.0/ - */ - -import org.jetbrains.annotations.NotNull; - -import java.io.ObjectStreamField; -import java.io.Serializable; -import java.lang.ThreadLocal; -import java.lang.reflect.ParameterizedType; -import java.lang.reflect.Type; -import java.util.Arrays; -import java.util.Collection; -import java.util.Iterator; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.LockSupport; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.IntFunction; - -/** - * Same as {@link ConcurrentHashMap}, but with primitive type int keys. - */ -@SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter") -public class ConcurrentIntHashMap implements Serializable { - static final int EMPTY_KEY = Integer.MIN_VALUE; - - /* - * Overview: - * - * The primary design goal of this hash table is to maintain - * concurrent readability (typically method get(), but also - * iterators and related methods) while minimizing update - * contention. Secondary goals are to keep space consumption about - * the same or better than java.util.HashMap, and to support high - * initial insertion rates on an empty table by many threads. - * - * This map usually acts as a binned (bucketed) hash table. Each - * key-value mapping is held in a Node. Most nodes are instances - * of the basic Node class with hash, key, value, and next - * fields. However, various subclasses exist: TreeNodes are - * arranged in balanced trees, not lists. TreeBins hold the roots - * of sets of TreeNodes. ForwardingNodes are placed at the heads - * of bins during resizing. ReservationNodes are used as - * placeholders while establishing values in computeIfAbsent and - * related methods. The types TreeBin, ForwardingNode, and - * ReservationNode do not hold normal user keys, values, or - * hashes, and are readily distinguishable during search etc - * because they have negative hash fields and null key and value - * fields. (These special nodes are either uncommon or transient, - * so the impact of carrying around some unused fields is - * insignificant.) - * - * The table is lazily initialized to a power-of-two size upon the - * first insertion. Each bin in the table normally contains a - * list of Nodes (most often, the list has only zero or one Node). - * Table accesses require volatile/atomic reads, writes, and - * CASes. Because there is no other way to arrange this without - * adding further indirections, we use intrinsics - * (sun.misc.Unsafe) operations. - * - * We use the top (sign) bit of Node hash fields for control - * purposes -- it is available anyway because of addressing - * constraints. Nodes with negative hash fields are specially - * handled or ignored in map methods. - * - * Insertion (via put or its variants) of the first node in an - * empty bin is performed by just CASing it to the bin. This is - * by far the most common case for put operations under most - * key/hash distributions. Other update operations (insert, - * delete, and replace) require locks. We do not want to waste - * the space required to associate a distinct lock object with - * each bin, so instead use the first node of a bin list itself as - * a lock. Locking support for these locks relies on builtin - * "synchronized" monitors. - * - * Using the first node of a list as a lock does not by itself - * suffice though: When a node is locked, any update must first - * validate that it is still the first node after locking it, and - * retry if not. Because new nodes are always appended to lists, - * once a node is first in a bin, it remains first until deleted - * or the bin becomes invalidated (upon resizing). - * - * The main disadvantage of per-bin locks is that other update - * operations on other nodes in a bin list protected by the same - * lock can stall, for example when user equals() or mapping - * functions take a long time. However, statistically, under - * random hash codes, this is not a common problem. Ideally, the - * frequency of nodes in bins follows a Poisson distribution - * (http://en.wikipedia.org/wiki/Poisson_distribution) with a - * parameter of about 0.5 on average, given the resizing threshold - * of 0.75, although with a large variance because of resizing - * granularity. Ignoring variance, the expected occurrences of - * list size k are (exp(-0.5) * pow(0.5, k) / factorial(k)). The - * first values are: - * - * 0: 0.60653066 - * 1: 0.30326533 - * 2: 0.07581633 - * 3: 0.01263606 - * 4: 0.00157952 - * 5: 0.00015795 - * 6: 0.00001316 - * 7: 0.00000094 - * 8: 0.00000006 - * more: less than 1 in ten million - * - * Lock contention probability for two threads accessing distinct - * elements is roughly 1 / (8 * #elements) under random hashes. - * - * Actual hash code distributions encountered in practice - * sometimes deviate significantly from uniform randomness. This - * includes the case when N > (1<<30), so some keys MUST collide. - * Similarly for dumb or hostile usages in which multiple keys are - * designed to have identical hash codes or ones that differs only - * in masked-out high bits. So we use a secondary strategy that - * applies when the number of nodes in a bin exceeds a - * threshold. These TreeBins use a balanced tree to hold nodes (a - * specialized form of red-black trees), bounding search time to - * O(log N). Each search step in a TreeBin is at least twice as - * slow as in a regular list, but given that N cannot exceed - * (1<<64) (before running out of addresses) this bounds search - * steps, lock hold times, etc, to reasonable constants (roughly - * 100 nodes inspected per operation worst case) so long as keys - * are Comparable (which is very common -- String, Long, etc). - * TreeBin nodes (TreeNodes) also maintain the same "next" - * traversal pointers as regular nodes, so can be traversed in - * iterators in the same way. - * - * The table is resized when occupancy exceeds a percentage - * threshold (nominally, 0.75, but see below). Any thread - * noticing an overfull bin may assist in resizing after the - * initiating thread allocates and sets up the replacement array. - * However, rather than stalling, these other threads may proceed - * with insertions etc. The use of TreeBins shields us from the - * worst case effects of overfilling while resizes are in - * progress. Resizing proceeds by transferring bins, one by one, - * from the table to the next table. However, threads claim small - * blocks of indices to transfer (via field transferIndex) before - * doing so, reducing contention. A generation stamp in field - * sizeCtl ensures that resizings do not overlap. Because we are - * using power-of-two expansion, the elements from each bin must - * either stay at same index, or move with a power of two - * offset. We eliminate unnecessary node creation by catching - * cases where old nodes can be reused because their next fields - * won't change. On average, only about one-sixth of them need - * cloning when a table doubles. The nodes they replace will be - * garbage collectable as soon as they are no longer referenced by - * any reader thread that may be in the midst of concurrently - * traversing table. Upon transfer, the old table bin contains - * only a special forwarding node (with hash field "MOVED") that - * contains the next table as its key. On encountering a - * forwarding node, access and update operations restart, using - * the new table. - * - * Each bin transfer requires its bin lock, which can stall - * waiting for locks while resizing. However, because other - * threads can join in and help resize rather than contend for - * locks, average aggregate waits become shorter as resizing - * progresses. The transfer operation must also ensure that all - * accessible bins in both the old and new table are usable by any - * traversal. This is arranged in part by proceeding from the - * last bin (table.length - 1) up towards the first. Upon seeing - * a forwarding node, traversals (see class Traverser) arrange to - * move to the new table without revisiting nodes. To ensure that - * no intervening nodes are skipped even when moved out of order, - * a stack (see class TableStack) is created on first encounter of - * a forwarding node during a traversal, to maintain its place if - * later processing the current table. The need for these - * save/restore mechanics is relatively rare, but when one - * forwarding node is encountered, typically many more will be. - * So Traversers use a simple caching scheme to avoid creating so - * many new TableStack nodes. (Thanks to Peter Levart for - * suggesting use of a stack here.) - * - * The traversal scheme also applies to partial traversals of - * ranges of bins (via an alternate Traverser constructor) - * to support partitioned aggregate operations. Also, read-only - * operations give up if ever forwarded to a null table, which - * provides support for shutdown-style clearing, which is also not - * currently implemented. - * - * Lazy table initialization minimizes footprint until first use, - * and also avoids resizings when the first operation is from a - * putAll, constructor with map argument, or deserialization. - * These cases attempt to override the initial capacity settings, - * but harmlessly fail to take effect in cases of races. - * - * The element count is maintained using a specialization of - * LongAdder. We need to incorporate a specialization rather than - * just use a LongAdder in order to access implicit - * contention-sensing that leads to creation of multiple - * CounterCells. The counter mechanics avoid contention on - * updates but can encounter cache thrashing if read too - * frequently during concurrent access. To avoid reading so often, - * resizing under contention is attempted only upon adding to a - * bin already holding two or more nodes. Under uniform hash - * distributions, the probability of this occurring at threshold - * is around 13%, meaning that only about 1 in 8 puts check - * threshold (and after resizing, many fewer do so). - * - * TreeBins use a special form of comparison for search and - * related operations (which is the main reason we cannot use - * existing collections such as TreeMaps). TreeBins contain - * Comparable elements, but may contain others, as well as - * elements that are Comparable but not necessarily Comparable for - * the same T, so we cannot invoke compareTo among them. To handle - * this, the tree is ordered primarily by hash value, then by - * Comparable.compareTo order if applicable. On lookup at a node, - * if elements are not comparable or compare as 0 then both left - * and right children may need to be searched in the case of tied - * hash values. (This corresponds to the full list search that - * would be necessary if all elements were non-Comparable and had - * tied hashes.) On insertion, to keep a total ordering (or as - * close as is required here) across rebalancings, we compare - * classes and identityHashCodes as tie-breakers. The red-black - * balancing code is updated from pre-jdk-collections - * (http://gee.cs.oswego.edu/dl/classes/collections/RBCell.java) - * based in turn on Cormen, Leiserson, and Rivest "Introduction to - * Algorithms" (CLR). - * - * TreeBins also require an additional locking mechanism. While - * list traversal is always possible by readers even during - * updates, tree traversal is not, mainly because of tree-rotations - * that may change the root node and/or its linkages. TreeBins - * include a simple read-write lock mechanism parasitic on the - * main bin-synchronization strategy: Structural adjustments - * associated with an insertion or removal are already bin-locked - * (and so cannot conflict with other writers) but must wait for - * ongoing readers to finish. Since there can be only one such - * waiter, we use a simple scheme using a single "waiter" field to - * block writers. However, readers need never block. If the root - * lock is held, they proceed along the slow traversal path (via - * next-pointers) until the lock becomes available or the list is - * exhausted, whichever comes first. These cases are not fast, but - * maximize aggregate expected throughput. - * - * Maintaining API and serialization compatibility with previous - * versions of this class introduces several oddities. Mainly: We - * leave untouched but unused constructor arguments referring to - * concurrencyLevel. We accept a loadFactor constructor argument, - * but apply it only to initial table capacity (which is the only - * time that we can guarantee to honor it.) We also declare an - * unused "Segment" class that is instantiated in minimal form - * only when serializing. - * - * Also, solely for compatibility with previous versions of this - * class, it extends AbstractMap, even though all of its methods - * are overridden, so it is just useless baggage. - * - * This file is organized to make things a little easier to follow - * while reading than they might otherwise: First the main static - * declarations and utilities, then fields, then main public - * methods (with a few factorings of multiple public methods into - * internal ones), then sizing methods, trees, traversers, and - * bulk operations. - */ - - /* ---------------- Constants -------------- */ - static final int HASH_BITS = 0x7fffffff; // usable bits of normal node hash - /** - * The largest possible (non-power of two) array size. - * Needed by toArray and related methods. - */ - static final int MAX_ARRAY_SIZE = Integer.MAX_VALUE - 8; - /** - * The smallest table capacity for which bins may be treeified. - * (Otherwise the table is resized if too many nodes in a bin.) - * The value should be at least 4 * TREEIFY_THRESHOLD to avoid - * conflicts between resizing and treeification thresholds. - */ - static final int MIN_TREEIFY_CAPACITY = 64; - /* - * Encodings for Node hash fields. See above for explanation. - */ - static final int MOVED = -1; // hash for forwarding nodes - /** - * Number of CPUS, to place bounds on some sizings - */ - static final int NCPU = Runtime.getRuntime().availableProcessors(); - static final int RESERVED = -3; // hash for transient reservations - static final int TREEBIN = -2; // hash for roots of trees - /** - * The bin count threshold for using a tree rather than list for a - * bin. Bins are converted to trees when adding an element to a - * bin with at least this many nodes. The value must be greater - * than 2, and should be at least 8 to mesh with assumptions in - * tree removal about conversion back to plain bins upon - * shrinkage. - */ - static final int TREEIFY_THRESHOLD = 8; - /** - * The bin count threshold for untreeifying a (split) bin during a - * resize operation. Should be less than TREEIFY_THRESHOLD, and at - * most 6 to mesh with shrinkage detection under removal. - */ - static final int UNTREEIFY_THRESHOLD = 6; - /* ---------------- Fields -------------- */ - private static final long ABASE; - private static final int ASHIFT; - /* - * Volatile access methods are used for table elements as well as - * elements of in-progress next table while resizing. All uses of - * the tab arguments must be null checked by callers. All callers - * also paranoically precheck that tab's length is not zero (or an - * equivalent check), thus ensuring that any index argument taking - * the form of a hash value anded with (length - 1) is a valid - * index. Note that, to be correct wrt arbitrary concurrency - * errors by users, these checks must operate on local variables, - * which accounts for some odd-looking inline assignments below. - * Note that calls to setTabAt always occur within locked regions, - * and so in principle require only release ordering, not - * full volatile semantics, but are currently coded as volatile - * writes to be conservative. - */ - private static final long BASECOUNT; - private static final long CELLSBUSY; - private static final long CELLVALUE; - /** - * The default initial table capacity. Must be a power of 2 - * (i.e., at least 1) and at most MAXIMUM_CAPACITY. - */ - private static final int DEFAULT_CAPACITY = 16; - /** - * The load factor for this table. Overrides of this value in - * constructors affect only the initial table capacity. The - * actual floating point value isn't normally used -- it is - * simpler to use expressions such as {@code n - (n >>> 2)} for - * the associated resizing threshold. - */ - private static final float LOAD_FACTOR = 0.75f; - /** - * The largest possible table capacity. This value must be - * exactly 1<<30 to stay within Java array allocation and indexing - * bounds for power of two table sizes, and is further required - * because the top two bits of 32bit hash fields are used for - * control purposes. - */ - private static final int MAXIMUM_CAPACITY = 1 << 30; - /** - * Minimum number of rebinnings per transfer step. Ranges are - * subdivided to allow multiple resizer threads. This value - * serves as a lower bound to avoid resizers encountering - * excessive memory contention. The value should be at least - * DEFAULT_CAPACITY. - */ - private static final int MIN_TRANSFER_STRIDE = 16; - private static final long PROBE; - - /* ---------------- Nodes -------------- */ - /** - * The increment for generating probe values - */ - private static final int PROBE_INCREMENT = 0x9e3779b9; - - /* ---------------- Static utilities -------------- */ - /** - * The number of bits used for generation stamp in sizeCtl. - * Must be at least 6 for 32bit arrays. - */ - private static final int RESIZE_STAMP_BITS = 16; - /** - * The maximum number of threads that can help resize. - * Must fit in 32 - RESIZE_STAMP_BITS bits. - */ - private static final int MAX_RESIZERS = (1 << (32 - RESIZE_STAMP_BITS)) - 1; - /** - * The bit shift for recording size stamp in sizeCtl. - */ - private static final int RESIZE_STAMP_SHIFT = 32 - RESIZE_STAMP_BITS; - private static final long SEED; - - /* ---------------- Table element access -------------- */ - /** - * The increment of seeder per new instance - */ - private static final long SEEDER_INCREMENT = 0xbb67ae8584caa73bL; - private static final long SIZECTL; - private static final long TRANSFERINDEX; - /** - * Generates per-thread initialization/probe field - */ - private static final AtomicInteger probeGenerator = new AtomicInteger(); - /** - * The next seed for default constructors. - */ - private static final AtomicLong seeder = new AtomicLong(initialSeed()); - /** - * For serialization compatibility. - */ - private static final ObjectStreamField[] serialPersistentFields = { - new ObjectStreamField("segments", Segment[].class), - new ObjectStreamField("segmentMask", Integer.TYPE), - new ObjectStreamField("segmentShift", Integer.TYPE) - }; - private static final long serialVersionUID = 7249069246763182397L; - private final ThreadLocal> tlTraverser = ThreadLocal.withInitial(Traverser::new); - /** - * The array of bins. Lazily initialized upon first insertion. - * Size is always a power of two. Accessed directly by iterators. - */ - transient volatile Node[] table; - /** - * Base counter value, used mainly when there is no contention, - * but also as a fallback during table initialization - * races. Updated via CAS. - */ - private transient volatile long baseCount; - /** - * Spinlock (locked via CAS) used when resizing and/or creating CounterCells. - */ - private transient volatile int cellsBusy; - /** - * Table of counter cells. When non-null, size is a power of 2. - */ - private transient volatile CounterCell[] counterCells; - // Original (since JDK1.2) Map methods - private transient EntrySetView entrySet; - /* ---------------- Public operations -------------- */ - // views - private transient KeySetView keySet; - /** - * The next table to use; non-null only while resizing. - */ - private transient volatile Node[] nextTable; - /** - * Table initialization and resizing control. When negative, the - * table is being initialized or resized: -1 for initialization, - * else -(1 + the number of active resizing threads). Otherwise, - * when table is null, holds the initial table size to use upon - * creation, or 0 for default. After initialization, holds the - * next element count value upon which to resize the table. - */ - private transient volatile int sizeCtl; - /** - * The next table index (plus one) to split while resizing. - */ - private transient volatile int transferIndex; - private transient ValuesView values; - - /** - * Creates a new, empty map with an initial table size based on - * the given number of elements ({@code initialCapacity}), table - * density ({@code loadFactor}), and number of concurrently - * updating threads ({@code concurrencyLevel}). - * - * @param initialCapacity the initial capacity. The implementation - * performs internal sizing to accommodate this many elements, - * given the specified load factor. - * @param loadFactor the load factor (table density) for - * establishing the initial table size - * @throws IllegalArgumentException if the initial capacity is - * negative or the load factor or concurrencyLevel are - * nonpositive - */ - public ConcurrentIntHashMap(int initialCapacity, float loadFactor) { - if (!(loadFactor > 0.0f) || initialCapacity < 0) - throw new IllegalArgumentException(); - if (initialCapacity < 1) // Use at least as many bins - initialCapacity = 1; // as estimated threads - long size = (long) (1.0 + (long) initialCapacity / loadFactor); - this.sizeCtl = (size >= (long) MAXIMUM_CAPACITY) ? - MAXIMUM_CAPACITY : tableSizeFor((int) size); - } - - /** - * Creates a new map with the same mappings as the given map. - * - * @param m the map - */ - public ConcurrentIntHashMap(ConcurrentIntHashMap m) { - this.sizeCtl = DEFAULT_CAPACITY; - putAll(m); - } - - /** - * Creates a new, empty map with the default initial table size (16). - */ - public ConcurrentIntHashMap() { - } - - /** - * Creates a new, empty map with an initial table size - * accommodating the specified number of elements without the need - * to dynamically resize. - * - * @param initialCapacity The implementation performs internal - * sizing to accommodate this many elements. - * @throws IllegalArgumentException if the initial capacity of - * elements is negative - */ - public ConcurrentIntHashMap(int initialCapacity) { - if (initialCapacity < 0) - throw new IllegalArgumentException(); - this.sizeCtl = ((initialCapacity >= (MAXIMUM_CAPACITY >>> 1)) ? - MAXIMUM_CAPACITY : - tableSizeFor(initialCapacity + (initialCapacity >>> 1) + 1)); - } - - /** - * Creates a new {@link Set} backed by a ConcurrentLongHashMap - * from the given type to {@code Boolean.TRUE}. - * - * @return the new set - * @since 1.8 - */ - public static KeySetView newKeySet() { - return new KeySetView<>(new ConcurrentIntHashMap<>(), Boolean.TRUE); - } - - /** - * Creates a new {@link Set} backed by a ConcurrentLongHashMap - * from the given type to {@code Boolean.TRUE}. - * - * @param initialCapacity The implementation performs internal - * sizing to accommodate this many elements. - * @return the new set - * @throws IllegalArgumentException if the initial capacity of - * elements is negative - * @since 1.8 - */ - public static KeySetView newKeySet(int initialCapacity) { - return new KeySetView<>(new ConcurrentIntHashMap<>(initialCapacity), Boolean.TRUE); - } - - /** - * Removes all of the mappings from this map. - */ - public void clear() { - long delta = 0L; // negative number of deletions - int i = 0; - Node[] tab = table; - while (tab != null && i < tab.length) { - int fh; - Node f = tabAt(tab, i); - if (f == null) - ++i; - else if ((fh = f.hash) == MOVED) { - tab = helpTransfer(tab, f); - i = 0; // restart - } else { - synchronized (f) { - if (tabAt(tab, i) == f) { - Node p = (fh >= 0 ? f : - (f instanceof TreeBin) ? - ((TreeBin) f).first : null); - while (p != null) { - --delta; - p = p.next; - } - setTabAt(tab, i++, null); - } - } - } - } - if (delta != 0L) - addCount(delta, -1); - } - - /** - * Attempts to compute a mapping for the specified key and its - * current mapped value (or {@code null} if there is no current - * mapping). The entire method invocation is performed atomically. - * Some attempted update operations on this map by other threads - * may be blocked while computation is in progress, so the - * computation should be short and simple, and must not attempt to - * update any other mappings of this Map. - * - * @param key key with which the specified value is to be associated - * @param remappingFunction the function to compute a value - * @return the new value associated with the specified key, or null if none - * @throws IllegalArgumentException if the specified key is negative - * @throws NullPointerException if the specified remappingFunction is null - * @throws IllegalStateException if the computation detectably - * attempts a recursive update to this map that would - * otherwise never complete - * @throws RuntimeException or Error if the remappingFunction does so, - * in which case the mapping is unchanged - */ - public V compute(int key, BiIntFunction remappingFunction) { - if (key < 0) - throw new IllegalArgumentException(); - if (remappingFunction == null) - throw new NullPointerException(); - int h = spread(keyHashCode(key)); - V val = null; - int delta = 0; - int binCount = 0; - for (Node[] tab = table; ; ) { - Node f; - int n, i, fh; - if (tab == null || (n = tab.length) == 0) - tab = initTable(); - else if ((f = tabAt(tab, i = (n - 1) & h)) == null) { - Node r = new ReservationNode<>(); - synchronized (r) { - if (casTabAt(tab, i, r)) { - binCount = 1; - Node node = null; - try { - if ((val = remappingFunction.apply(key, null)) != null) { - delta = 1; - node = new Node<>(h, key, val, null); - } - } finally { - setTabAt(tab, i, node); - } - } - } - if (binCount != 0) - break; - } else if ((fh = f.hash) == MOVED) - tab = helpTransfer(tab, f); - else { - synchronized (f) { - if (tabAt(tab, i) == f) { - if (fh >= 0) { - binCount = 1; - for (Node e = f, pred = null; ; ++binCount) { - if (e.hash == h && (e.key == key)) { - val = remappingFunction.apply(key, e.val); - if (val != null) - e.val = val; - else { - delta = -1; - Node en = e.next; - if (pred != null) - pred.next = en; - else - setTabAt(tab, i, en); - } - break; - } - pred = e; - if ((e = e.next) == null) { - val = remappingFunction.apply(key, null); - if (val != null) { - delta = 1; - pred.next = new Node<>(h, key, val, null); - } - break; - } - } - } else if (f instanceof TreeBin) { - binCount = 1; - TreeBin t = (TreeBin) f; - TreeNode r, p; - if ((r = t.root) != null) - p = r.findTreeNode(h, key); - else - p = null; - V pv = (p == null) ? null : p.val; - val = remappingFunction.apply(key, pv); - if (val != null) { - if (p != null) - p.val = val; - else { - delta = 1; - t.putTreeVal(h, key, val); - } - } else if (p != null) { - delta = -1; - if (t.removeTreeNode(p)) - setTabAt(tab, i, untreeify(t.first)); - } - } - } - } - if (binCount != 0) { - if (binCount >= TREEIFY_THRESHOLD) - treeifyBin(tab, i); - break; - } - } - } - if (delta != 0) - addCount(delta, binCount); - return val; - } - - /** - * If the specified key is not already associated with a value, - * attempts to compute its value using the given mapping function - * and enters it into this map unless {@code null}. The entire - * method invocation is performed atomically, so the function is - * applied at most once per key. Some attempted update operations - * on this map by other threads may be blocked while computation - * is in progress, so the computation should be short and simple, - * and must not attempt to update any other mappings of this map. - * - * @param key key with which the specified value is to be associated - * @param token token to pass to the mapping function - * @param mappingFunction the function to compute a value - * @return the current (existing or computed) value associated with - * the specified key, or null if the computed value is null - * @throws IllegalArgumentException if the specified key is negative - * @throws NullPointerException if the specified mappingFunction is null - * @throws IllegalStateException if the computation detectably - * attempts a recursive update to this map that would - * otherwise never complete - * @throws RuntimeException or Error if the mappingFunction does so, - * in which case the mapping is left unestablished - */ - public V computeIfAbsent(int key, Object token, BiIntFunction mappingFunction) { - if (key < 0) - throw new IllegalArgumentException(); - if (mappingFunction == null) - throw new NullPointerException(); - int h = spread(keyHashCode(key)); - V val = null; - int binCount = 0; - for (Node[] tab = table; ; ) { - Node f; - int n, i, fh; - if (tab == null || (n = tab.length) == 0) - tab = initTable(); - else if ((f = tabAt(tab, i = (n - 1) & h)) == null) { - Node r = new ReservationNode<>(); - synchronized (r) { - if (casTabAt(tab, i, r)) { - binCount = 1; - Node node = null; - try { - if ((val = mappingFunction.apply(key, token)) != null) - node = new Node<>(h, key, val, null); - } finally { - setTabAt(tab, i, node); - } - } - } - if (binCount != 0) - break; - } else if ((fh = f.hash) == MOVED) - tab = helpTransfer(tab, f); - else { - boolean added = false; - synchronized (f) { - if (tabAt(tab, i) == f) { - if (fh >= 0) { - binCount = 1; - for (Node e = f; ; ++binCount) { - if (e.hash == h && e.key == key) { - val = e.val; - break; - } - Node pred = e; - if ((e = e.next) == null) { - if ((val = mappingFunction.apply(key, token)) != null) { - added = true; - pred.next = new Node<>(h, key, val, null); - } - break; - } - } - } else if (f instanceof TreeBin) { - binCount = 2; - TreeBin t = (TreeBin) f; - TreeNode r, p; - if ((r = t.root) != null && - (p = r.findTreeNode(h, key)) != null) - val = p.val; - else if ((val = mappingFunction.apply(key, token)) != null) { - added = true; - t.putTreeVal(h, key, val); - } - } - } - } - if (binCount != 0) { - if (binCount >= TREEIFY_THRESHOLD) - treeifyBin(tab, i); - if (!added) - return val; - break; - } - } - } - if (val != null) - addCount(1L, binCount); - return val; - } - - /** - * If the specified key is not already associated with a value, - * attempts to compute its value using the given mapping function - * and enters it into this map unless {@code null}. The entire - * method invocation is performed atomically, so the function is - * applied at most once per key. Some attempted update operations - * on this map by other threads may be blocked while computation - * is in progress, so the computation should be short and simple, - * and must not attempt to update any other mappings of this map. - * - * @param key key with which the specified value is to be associated - * @param mappingFunction the function to compute a value - * @return the current (existing or computed) value associated with - * the specified key, or null if the computed value is null - * @throws IllegalArgumentException if the specified key is negative - * @throws NullPointerException if the specified key or mappingFunction - * is null - * @throws IllegalStateException if the computation detectably - * attempts a recursive update to this map that would - * otherwise never complete - * @throws RuntimeException or Error if the mappingFunction does so, - * in which case the mapping is left unestablished - */ - public V computeIfAbsent(int key, IntFunction mappingFunction) { - if (key < 0) - throw new IllegalArgumentException(); - if (mappingFunction == null) - throw new NullPointerException(); - int h = spread(keyHashCode(key)); - V val = null; - int binCount = 0; - for (Node[] tab = table; ; ) { - Node f; - int n, i, fh; - if (tab == null || (n = tab.length) == 0) - tab = initTable(); - else if ((f = tabAt(tab, i = (n - 1) & h)) == null) { - Node r = new ReservationNode<>(); - synchronized (r) { - if (casTabAt(tab, i, r)) { - binCount = 1; - Node node = null; - try { - if ((val = mappingFunction.apply(key)) != null) - node = new Node<>(h, key, val, null); - } finally { - setTabAt(tab, i, node); - } - } - } - if (binCount != 0) - break; - } else if ((fh = f.hash) == MOVED) - tab = helpTransfer(tab, f); - else { - boolean added = false; - synchronized (f) { - if (tabAt(tab, i) == f) { - if (fh >= 0) { - binCount = 1; - for (Node e = f; ; ++binCount) { - if (e.hash == h && e.key == key) { - val = e.val; - break; - } - Node pred = e; - if ((e = e.next) == null) { - if ((val = mappingFunction.apply(key)) != null) { - added = true; - pred.next = new Node<>(h, key, val, null); - } - break; - } - } - } else if (f instanceof TreeBin) { - binCount = 2; - TreeBin t = (TreeBin) f; - TreeNode r, p; - if ((r = t.root) != null && - (p = r.findTreeNode(h, key)) != null) - val = p.val; - else if ((val = mappingFunction.apply(key)) != null) { - added = true; - t.putTreeVal(h, key, val); - } - } - } - } - if (binCount != 0) { - if (binCount >= TREEIFY_THRESHOLD) - treeifyBin(tab, i); - if (!added) - return val; - break; - } - } - } - if (val != null) - addCount(1L, binCount); - return val; - } - - /** - * If the value for the specified key is present, attempts to - * compute a new mapping given the key and its current mapped - * value. The entire method invocation is performed atomically. - * Some attempted update operations on this map by other threads - * may be blocked while computation is in progress, so the - * computation should be short and simple, and must not attempt to - * update any other mappings of this map. - * - * @param key key with which a value may be associated - * @param remappingFunction the function to compute a value - * @return the new value associated with the specified key, or null if none - * @throws IllegalArgumentException if the specified key is negative - * @throws NullPointerException if the specified remappingFunction is null - * @throws IllegalStateException if the computation detectably - * attempts a recursive update to this map that would - * otherwise never complete - * @throws RuntimeException or Error if the remappingFunction does so, - * in which case the mapping is unchanged - */ - public V computeIfPresent(int key, BiIntFunction remappingFunction) { - if (key < 0) - throw new IllegalArgumentException(); - if (remappingFunction == null) - throw new NullPointerException(); - int h = spread(keyHashCode(key)); - V val = null; - int delta = 0; - int binCount = 0; - for (Node[] tab = table; ; ) { - Node f; - int n, i, fh; - if (tab == null || (n = tab.length) == 0) - tab = initTable(); - else if ((f = tabAt(tab, i = (n - 1) & h)) == null) - break; - else if ((fh = f.hash) == MOVED) - tab = helpTransfer(tab, f); - else { - synchronized (f) { - if (tabAt(tab, i) == f) { - if (fh >= 0) { - binCount = 1; - for (Node e = f, pred = null; ; ++binCount) { - if (e.hash == h && e.key == key) { - val = remappingFunction.apply(key, e.val); - if (val != null) - e.val = val; - else { - delta = -1; - Node en = e.next; - if (pred != null) - pred.next = en; - else - setTabAt(tab, i, en); - } - break; - } - pred = e; - if ((e = e.next) == null) - break; - } - } else if (f instanceof TreeBin) { - binCount = 2; - TreeBin t = (TreeBin) f; - TreeNode r, p; - if ((r = t.root) != null && - (p = r.findTreeNode(h, key)) != null) { - val = remappingFunction.apply(key, p.val); - if (val != null) - p.val = val; - else { - delta = -1; - if (t.removeTreeNode(p)) - setTabAt(tab, i, untreeify(t.first)); - } - } - } - } - } - if (binCount != 0) - break; - } - } - if (delta != 0) - addCount(delta, binCount); - return val; - } - - /** - * Tests if the specified object is a key in this table. - * - * @param key possible key - * @return {@code true} if and only if the specified object - * is a key in this table, as determined by the - * {@code equals} method; {@code false} otherwise - * @throws NullPointerException if the specified key is null - */ - public boolean containsKey(int key) { - return get(key) != null; - } - - /** - * Returns {@code true} if this map maps one or more keys to the - * specified value. Note: This method may require a full traversal - * of the map, and is much slower than method {@code containsKey}. - * - * @param value value whose presence in this map is to be tested - * @return {@code true} if this map maps one or more keys to the - * specified value - * @throws NullPointerException if the specified value is null - */ - public boolean containsValue(V value) { - if (value == null) - throw new NullPointerException(); - Node[] t = table; - if (t != null) { - Traverser it = getTraverser(t); - for (Node p; (p = it.advance()) != null; ) { - V v; - if ((v = p.val) == value || (value.equals(v))) - return true; - } - } - return false; - } - - /** - * Returns a {@link Set} view of the mappings contained in this map. - * The set is backed by the map, so changes to the map are - * reflected in the set, and vice-versa. The set supports element - * removal, which removes the corresponding mapping from the map, - * via the {@code Iterator.remove}, {@code Set.remove}, - * {@code removeAll}, {@code retainAll}, and {@code clear} - * operations. - *

    The view's iterators and spliterators are - * weakly consistent. - * - * @return the set view - */ - @NotNull - public Set> entrySet() { - EntrySetView es; - return (es = entrySet) != null ? es : (entrySet = new EntrySetView<>(this)); - } - - /** - * Compares the specified object with this map for equality. - * Returns {@code true} if the given object is a map with the same - * mappings as this map. This operation may return misleading - * results if either map is concurrently modified during execution - * of this method. - * - * @param o object to be compared for equality with this map - * @return {@code true} if the specified object is equal to this map - */ - public boolean equals(Object o) { - if (o != this) { - if (!(o instanceof ConcurrentIntHashMap)) - return false; - ConcurrentIntHashMap m = (ConcurrentIntHashMap) o; - Traverser it = getTraverser(table); - for (Node p; (p = it.advance()) != null; ) { - V val = p.val; - Object v = m.get(p.key); - if (v == null || (v != val && !v.equals(val))) - return false; - } - for (IntEntry e : m.entrySet()) { - int mk; - Object mv, v; - if ((mk = e.getKey()) == EMPTY_KEY || - (mv = e.getValue()) == null || - (v = get(mk)) == null || - (mv != v && !mv.equals(v))) - return false; - } - } - return true; - } - - /** - * Returns the value to which the specified key is mapped, - * or {@code null} if this map contains no mapping for the key. - *

    More formally, if this map contains a mapping from a key - * {@code k} to a value {@code v} such that {@code key.equals(k)}, - * then this method returns {@code v}; otherwise it returns - * {@code null}. (There can be at most one such mapping.) - * - * @param key map key value - * @return value to which specified key is mapped - * @throws NullPointerException if the specified key is null - */ - public V get(int key) { - Node[] tab; - Node e, p; - int n, eh; - int h = spread(keyHashCode(key)); - if ((tab = table) != null && (n = tab.length) > 0 && - (e = tabAt(tab, (n - 1) & h)) != null) { - if ((eh = e.hash) == h) { - if (e.key == key) - return e.val; - } else if (eh < 0) - return (p = e.find(h, key)) != null ? p.val : null; - while ((e = e.next) != null) { - if (e.hash == h && e.key == key) - return e.val; - } - } - return null; - } - - /** - * Returns the value to which the specified key is mapped, or the - * given default value if this map contains no mapping for the - * key. - * - * @param key the key whose associated value is to be returned - * @param defaultValue the value to return if this map contains - * no mapping for the given key - * @return the mapping for the key, if present; else the default value - * @throws NullPointerException if the specified key is null - */ - public V getOrDefault(int key, V defaultValue) { - V v; - return (v = get(key)) == null ? defaultValue : v; - } - - /** - * Returns the hash code value for this {@link Map}, i.e., - * the sum of, for each key-value pair in the map, - * {@code key.hashCode() ^ value.hashCode()}. - * - * @return the hash code value for this map - */ - public int hashCode() { - int h = 0; - Node[] t = table; - if (t != null) { - Traverser it = getTraverser(t); - for (Node p; (p = it.advance()) != null; ) - h += keyHashCode(p.key) ^ p.val.hashCode(); - } - return h; - } - - // ConcurrentMap methods - - /** - * {@inheritDoc} - */ - public boolean isEmpty() { - return sumCount() <= 0L; // ignore transient negative values - } - - /** - * Returns a {@link Set} view of the keys in this map, using the - * given common mapped value for any additions (i.e., {@link - * Collection#add} and {@link Collection#addAll(Collection)}). - * This is of course only appropriate if it is acceptable to use - * the same value for all additions from this view. - * - * @param mappedValue the mapped value to use for any additions - * @return the set view - * @throws NullPointerException if the mappedValue is null - */ - public KeySetView keySet(V mappedValue) { - if (mappedValue == null) - throw new NullPointerException(); - return new KeySetView<>(this, mappedValue); - } - - /** - * Returns a {@link Set} view of the keys contained in this map. - * The set is backed by the map, so changes to the map are - * reflected in the set, and vice-versa. The set supports element - * removal, which removes the corresponding mapping from this map, - * via the {@code Iterator.remove}, {@code Set.remove}, - * {@code removeAll}, {@code retainAll}, and {@code clear} - * operations. It does not support the {@code add} or - * {@code addAll} operations. - *

    The view's iterators and spliterators are - * weakly consistent. - *

    - * - * @return the set view - */ - @NotNull - public KeySetView keySet() { - KeySetView ks; - return (ks = keySet) != null ? ks : (keySet = new KeySetView<>(this, null)); - } - - /** - * Returns the number of mappings. This method should be used - * instead of {@link #size} because a ConcurrentLongHashMap may - * contain more mappings than can be represented as an int. The - * value returned is an estimate; the actual count may differ if - * there are concurrent insertions or removals. - * - * @return the number of mappings - * @since 1.8 - */ - public long mappingCount() { - return Math.max(sumCount(), 0L); // ignore transient negative values - } - - // Overrides of JDK8+ Map extension method defaults - - /** - * Maps the specified key to the specified value in this table. - * Neither the key nor the value can be null. - *

    The value can be retrieved by calling the {@code get} method - * with a key that is equal to the original key. - * - * @param key key with which the specified value is to be associated - * @param value value to be associated with the specified key - * @return the previous value associated with {@code key}, or - * {@code null} if there was no mapping for {@code key} - * @throws NullPointerException if the specified key or value is null - */ - public V put(int key, V value) { - return putVal(key, value, false); - } - - /** - * Copies all of the mappings from the specified map to this one. - * These mappings replace any mappings that this map had for any of the - * keys currently in the specified map. - * - * @param m mappings to be stored in this map - */ - public void putAll(@NotNull ConcurrentIntHashMap m) { - tryPresize(m.size()); - for (IntEntry e : m.entrySet()) - putVal(e.getKey(), e.getValue(), false); - } - - /** - * {@inheritDoc} - * - * @return the previous value associated with the specified key, - * or {@code null} if there was no mapping for the key - * @throws NullPointerException if the specified key or value is null - */ - public V putIfAbsent(int key, V value) { - return putVal(key, value, true); - } - - public boolean remove(int key, V value) { - return value != null && replaceNode(key, null, value) != null; - } - - /** - * Removes the key (and its corresponding value) from this map. - * This method does nothing if the key is not in the map. - * - * @param key the key that needs to be removed - * @return the previous value associated with {@code key}, or - * {@code null} if there was no mapping for {@code key} - * @throws NullPointerException if the specified key is null - */ - public V remove(int key) { - return replaceNode(key, null, null); - } - - // Hashtable legacy methods - - public boolean replace(int key, @NotNull V oldValue, @NotNull V newValue) { - return replaceNode(key, newValue, oldValue) != null; - } - - // ConcurrentLongHashMap-only methods - - /** - * {@inheritDoc} - * - * @return the previous value associated with the specified key, - * or {@code null} if there was no mapping for the key - * @throws NullPointerException if the specified key or value is null - */ - public V replace(int key, @NotNull V value) { - return replaceNode(key, value, null); - } - - /** - * {@inheritDoc} - */ - public int size() { - long n = sumCount(); - return ((n < 0L) ? 0 : - (n > (long) Integer.MAX_VALUE) ? Integer.MAX_VALUE : - (int) n); - } - - /** - * Returns a string representation of this map. The string - * representation consists of a list of key-value mappings (in no - * particular order) enclosed in braces ("{@code {}}"). Adjacent - * mappings are separated by the characters {@code ", "} (comma - * and space). Each key-value mapping is rendered as the key - * followed by an equals sign ("{@code =}") followed by the - * associated value. - * - * @return a string representation of this map - */ - public String toString() { - Traverser it = getTraverser(table); - StringBuilder sb = new StringBuilder(); - sb.append('{'); - Node p; - if ((p = it.advance()) != null) { - for (; ; ) { - int k = p.key; - V v = p.val; - sb.append(k); - sb.append('='); - sb.append(v == this ? "(this Map)" : v); - if ((p = it.advance()) == null) - break; - sb.append(',').append(' '); - } - } - return sb.append('}').toString(); - } - - /** - * Returns a {@link Collection} view of the values contained in this map. - * The collection is backed by the map, so changes to the map are - * reflected in the collection, and vice-versa. The collection - * supports element removal, which removes the corresponding - * mapping from this map, via the {@code Iterator.remove}, - * {@code Collection.remove}, {@code removeAll}, - * {@code retainAll}, and {@code clear} operations. It does not - * support the {@code add} or {@code addAll} operations. - *

    The view's iterators and spliterators are - * weakly consistent. - * - * @return the collection view - */ - @NotNull - public Collection values() { - ValuesView vs; - return (vs = values) != null ? vs : (values = new ValuesView<>(this)); - } - - /* ---------------- Special Nodes -------------- */ - - private static long initialSeed() { - String pp = System.getProperty("java.util.secureRandomSeed"); - - if (pp != null && pp.equalsIgnoreCase("true")) { - byte[] seedBytes = java.security.SecureRandom.getSeed(8); - long s = (long) (seedBytes[0]) & 0xffL; - for (int i = 1; i < 8; ++i) - s = (s << 8) | ((long) (seedBytes[i]) & 0xffL); - return s; - } - return (mix64(System.currentTimeMillis()) ^ - mix64(System.nanoTime())); - } - - /* ---------------- Table Initialization and Resizing -------------- */ - - private static int keyHashCode(int key) { - return key; - } - - private static long mix64(long z) { - z = (z ^ (z >>> 33)) * 0xff51afd7ed558ccdL; - z = (z ^ (z >>> 33)) * 0xc4ceb9fe1a85ec53L; - return z ^ (z >>> 33); - } - - /** - * Returns a power of two table size for the given desired capacity. - * See Hackers Delight, sec 3.2 - */ - private static int tableSizeFor(int c) { - int n = c - 1; - n |= n >>> 1; - n |= n >>> 2; - n |= n >>> 4; - n |= n >>> 8; - n |= n >>> 16; - return (n < 0) ? 1 : (n >= MAXIMUM_CAPACITY) ? MAXIMUM_CAPACITY : n + 1; - } - - /** - * Adds to count, and if table is too small and not already - * resizing, initiates transfer. If already resizing, helps - * perform transfer if work is available. Rechecks occupancy - * after a transfer to see if another resize is already needed - * because resizings are lagging additions. - * - * @param x the count to add - * @param check if <0, don't check resize, if <= 1 only check if uncontended - */ - private void addCount(long x, int check) { - CounterCell[] as; - long b, s; - if ((as = counterCells) != null || !Unsafe.cas(this, BASECOUNT, b = baseCount, s = b + x)) { - CounterCell a; - long v; - int m; - boolean uncontended = true; - if (as == null || (m = as.length - 1) < 0 || - (a = as[getProbe() & m]) == null || - !(uncontended = Unsafe.cas(a, CELLVALUE, v = a.value, v + x))) { - fullAddCount(x, uncontended); - return; - } - if (check <= 1) - return; - s = sumCount(); - } - if (check >= 0) { - Node[] tab, nt; - int n, sc; - while (s >= (long) (sc = sizeCtl) && (tab = table) != null && - (n = tab.length) < MAXIMUM_CAPACITY) { - int rs = resizeStamp(n); - if (sc < 0) { - if (sc >>> RESIZE_STAMP_SHIFT != rs || sc == rs + MAX_RESIZERS || (nt = nextTable) == null || transferIndex <= 0) - break; - if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc, sc + 1)) - transfer(tab, nt); - } else if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc, - (rs << RESIZE_STAMP_SHIFT) + 2)) - transfer(tab, null); - s = sumCount(); - } - } - } - - // See LongAdder version for explanation - private void fullAddCount(long x, boolean wasUncontended) { - int h; - if ((h = getProbe()) == 0) { - localInit(); // force initialization - h = getProbe(); - wasUncontended = true; - } - boolean collide = false; // True if last slot nonempty - for (; ; ) { - CounterCell[] as; - CounterCell a; - int n; - long v; - if ((as = counterCells) != null && (n = as.length) > 0) { - if ((a = as[(n - 1) & h]) == null) { - if (cellsBusy == 0) { // Try to attach new Cell - CounterCell r = new CounterCell(x); // Optimistic create - if (cellsBusy == 0 && - Unsafe.getUnsafe().compareAndSwapInt(this, CELLSBUSY, 0, 1)) { - boolean created = false; - try { // Recheck under lock - CounterCell[] rs; - int m, j; - if ((rs = counterCells) != null && - (m = rs.length) > 0 && - rs[j = (m - 1) & h] == null) { - rs[j] = r; - created = true; - } - } finally { - cellsBusy = 0; - } - if (created) - break; - continue; // Slot is now non-empty - } - } - collide = false; - } else if (!wasUncontended) // CAS already known to fail - wasUncontended = true; // Continue after rehash - else if (Unsafe.cas(a, CELLVALUE, v = a.value, v + x)) - break; - else if (counterCells != as || n >= NCPU) - collide = false; // At max size or stale - else if (!collide) - collide = true; - else if (cellsBusy == 0 && - Unsafe.getUnsafe().compareAndSwapInt(this, CELLSBUSY, 0, 1)) { - try { - if (counterCells == as) {// Expand table unless stale - CounterCell[] rs = new CounterCell[n << 1]; - System.arraycopy(as, 0, rs, 0, n); - counterCells = rs; - } - } finally { - cellsBusy = 0; - } - collide = false; - continue; // Retry with expanded table - } - h = advanceProbe(h); - } else if (cellsBusy == 0 && counterCells == as && - Unsafe.getUnsafe().compareAndSwapInt(this, CELLSBUSY, 0, 1)) { - boolean init = false; - try { // Initialize table - if (counterCells == as) { - CounterCell[] rs = new CounterCell[2]; - rs[h & 1] = new CounterCell(x); - counterCells = rs; - init = true; - } - } finally { - cellsBusy = 0; - } - if (init) - break; - } else if (Unsafe.cas(this, BASECOUNT, v = baseCount, v + x)) - break; // Fall back on using base - } - } - - private Traverser getTraverser(Node[] tab) { - Traverser traverser = tlTraverser.get(); - int len = tab == null ? 0 : tab.length; - traverser.of(tab, len, len); - return traverser; - } - - /** - * Initializes table, using the size recorded in sizeCtl. - */ - private Node[] initTable() { - Node[] tab; - int sc; - while ((tab = table) == null || tab.length == 0) { - if ((sc = sizeCtl) < 0) - Os.pause(); // lost initialization race; just spin - else if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc, -1)) { - try { - if ((tab = table) == null || tab.length == 0) { - int n = (sc > 0) ? sc : DEFAULT_CAPACITY; - @SuppressWarnings("unchecked") - Node[] nt = (Node[]) new Node[n]; - table = tab = nt; - sc = n - (n >>> 2); - } - } finally { - sizeCtl = sc; - } - break; - } - } - return tab; - } - - /** - * Moves and/or copies the nodes in each bin to new table. See - * above for explanation. - */ - private void transfer(Node[] tab, Node[] nextTab) { - int n = tab.length, stride; - if ((stride = (NCPU > 1) ? (n >>> 3) / NCPU : n) < MIN_TRANSFER_STRIDE) - stride = MIN_TRANSFER_STRIDE; // subdivide range - if (nextTab == null) { // initiating - try { - @SuppressWarnings("unchecked") - Node[] nt = (Node[]) new Node[n << 1]; - nextTab = nt; - } catch (Throwable ex) { // try to cope with OOME - sizeCtl = Integer.MAX_VALUE; - return; - } - nextTable = nextTab; - transferIndex = n; - } - int nextn = nextTab.length; - ForwardingNode fwd = new ForwardingNode<>(nextTab); - boolean advance = true; - boolean finishing = false; // to ensure sweep before committing nextTab - for (int i = 0, bound = 0; ; ) { - Node f; - int fh; - while (advance) { - int nextIndex, nextBound; - if (--i >= bound || finishing) - advance = false; - else if ((nextIndex = transferIndex) <= 0) { - i = -1; - advance = false; - } else if (Unsafe.getUnsafe().compareAndSwapInt - (this, TRANSFERINDEX, nextIndex, - nextBound = (nextIndex > stride ? - nextIndex - stride : 0))) { - bound = nextBound; - i = nextIndex - 1; - advance = false; - } - } - if (i < 0 || i >= n || i + n >= nextn) { - int sc; - if (finishing) { - nextTable = null; - table = nextTab; - sizeCtl = (n << 1) - (n >>> 1); - return; - } - if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc = sizeCtl, sc - 1)) { - if ((sc - 2) != resizeStamp(n) << RESIZE_STAMP_SHIFT) - return; - finishing = advance = true; - i = n; // recheck before commit - } - } else if ((f = tabAt(tab, i)) == null) - advance = casTabAt(tab, i, fwd); - else if ((fh = f.hash) == MOVED) - advance = true; // already processed - else { - synchronized (f) { - if (tabAt(tab, i) == f) { - Node ln, hn; - if (fh >= 0) { - int runBit = fh & n; - Node lastRun = f; - for (Node p = f.next; p != null; p = p.next) { - int b = p.hash & n; - if (b != runBit) { - runBit = b; - lastRun = p; - } - } - if (runBit == 0) { - ln = lastRun; - hn = null; - } else { - hn = lastRun; - ln = null; - } - for (Node p = f; p != lastRun; p = p.next) { - int ph = p.hash; - int pk = p.key; - V pv = p.val; - if ((ph & n) == 0) - ln = new Node<>(ph, pk, pv, ln); - else - hn = new Node<>(ph, pk, pv, hn); - } - setTabAt(nextTab, i, ln); - setTabAt(nextTab, i + n, hn); - setTabAt(tab, i, fwd); - advance = true; - } else if (f instanceof TreeBin) { - TreeBin t = (TreeBin) f; - TreeNode lo = null, loTail = null; - TreeNode hi = null, hiTail = null; - int lc = 0, hc = 0; - for (Node e = t.first; e != null; e = e.next) { - int h = e.hash; - TreeNode p = new TreeNode<>(h, e.key, e.val, null, null); - if ((h & n) == 0) { - if ((p.prev = loTail) == null) - lo = p; - else - loTail.next = p; - loTail = p; - ++lc; - } else { - if ((p.prev = hiTail) == null) - hi = p; - else - hiTail.next = p; - hiTail = p; - ++hc; - } - } - ln = (lc <= UNTREEIFY_THRESHOLD) ? untreeify(lo) : - (hc != 0) ? new TreeBin<>(lo) : t; - hn = (hc <= UNTREEIFY_THRESHOLD) ? untreeify(hi) : - (lc != 0) ? new TreeBin<>(hi) : t; - setTabAt(nextTab, i, ln); - setTabAt(nextTab, i + n, hn); - setTabAt(tab, i, fwd); - advance = true; - } - } - } - } - } - } - /* ---------------- Counter support -------------- */ - - /** - * Replaces all linked nodes in bin at given index unless table is - * too small, in which case resizes instead. - */ - private void treeifyBin(Node[] tab, int index) { - Node b; - int n; - if (tab != null) { - if ((n = tab.length) < MIN_TREEIFY_CAPACITY) - tryPresize(n << 1); - else if ((b = tabAt(tab, index)) != null && b.hash >= 0) { - synchronized (b) { - if (tabAt(tab, index) == b) { - TreeNode hd = null, tl = null; - for (Node e = b; e != null; e = e.next) { - TreeNode p = - new TreeNode<>(e.hash, e.key, e.val, null, null); - if ((p.prev = tl) == null) - hd = p; - else - tl.next = p; - tl = p; - } - setTabAt(tab, index, new TreeBin<>(hd)); - } - } - } - } - } - - /** - * Tries to presize table to accommodate the given number of elements. - * - * @param size number of elements (doesn't need to be perfectly accurate) - */ - private void tryPresize(int size) { - int c = (size >= (MAXIMUM_CAPACITY >>> 1)) ? MAXIMUM_CAPACITY : - tableSizeFor(size + (size >>> 1) + 1); - int sc; - while ((sc = sizeCtl) >= 0) { - Node[] tab = table; - int n; - if (tab == null || (n = tab.length) == 0) { - n = Math.max(sc, c); - if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc, -1)) { - try { - if (table == tab) { - @SuppressWarnings("unchecked") - Node[] nt = (Node[]) new Node[n]; - table = nt; - sc = n - (n >>> 2); - } - } finally { - sizeCtl = sc; - } - } - } else if (c <= sc || n >= MAXIMUM_CAPACITY) - break; - else if (tab == table) { - int rs = resizeStamp(n); - if (sc < 0) { - Node[] nt; - if ((sc >>> RESIZE_STAMP_SHIFT) != rs || sc == rs + 1 || - sc == rs + MAX_RESIZERS || (nt = nextTable) == null || - transferIndex <= 0) - break; - if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc, sc + 1)) - transfer(tab, nt); - } else if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc, - (rs << RESIZE_STAMP_SHIFT) + 2)) - transfer(tab, null); - } - } - } - - static int advanceProbe(int probe) { - probe ^= probe << 13; // xorshift - probe ^= probe >>> 17; - probe ^= probe << 5; - Unsafe.getUnsafe().putInt(Thread.currentThread(), PROBE, probe); - return probe; - } - - /* ---------------- Conversion from/to TreeBins -------------- */ - - static boolean casTabAt(Node[] tab, int i, - Node v) { - return Unsafe.getUnsafe().compareAndSwapObject(tab, ((long) i << ASHIFT) + ABASE, null, v); - } - - /** - * Returns x's Class if it is of the form "class C implements - * Comparable", else null. - */ - static Class comparableClassFor(Object x) { - if (x instanceof Comparable) { - Class c; - Type[] ts, as; - Type t; - ParameterizedType p; - if ((c = x.getClass()) == String.class) // bypass checks - return c; - if ((ts = c.getGenericInterfaces()) != null) { - for (int i = 0; i < ts.length; ++i) { - if (((t = ts[i]) instanceof ParameterizedType) && - ((p = (ParameterizedType) t).getRawType() == - Comparable.class) && - (as = p.getActualTypeArguments()) != null && - as.length == 1 && as[0] == c) // type arg is c - return c; - } - } - } - return null; - } - - /* ---------------- TreeNodes -------------- */ - - /** - * Returns k.compareTo(x) if x matches kc (k's screened comparable - * class), else 0. - */ - static int compareComparables(int k, long x) { - return Long.compare(k, x); - } - - /* ---------------- TreeBins -------------- */ - - static int getProbe() { - return Unsafe.getUnsafe().getInt(Thread.currentThread(), PROBE); - } - - /* ----------------Table Traversal -------------- */ - - /** - * Initialize Thread fields for the current thread. Called only - * when Thread.threadLocalRandomProbe is zero, indicating that a - * thread local seed value needs to be generated. Note that even - * though the initialization is purely thread-local, we need to - * rely on (static) atomic generators to initialize the values. - */ - static void localInit() { - int p = probeGenerator.addAndGet(PROBE_INCREMENT); - int probe = (p == 0) ? 1 : p; // skip 0 - long seed = mix64(seeder.getAndAdd(SEEDER_INCREMENT)); - Thread t = Thread.currentThread(); - Unsafe.getUnsafe().putLong(t, SEED, seed); - Unsafe.getUnsafe().putInt(t, PROBE, probe); - } - - /** - * Returns the stamp bits for resizing a table of size n. - * Must be negative when shifted left by RESIZE_STAMP_SHIFT. - */ - static int resizeStamp(int n) { - return Integer.numberOfLeadingZeros(n) | (1 << (RESIZE_STAMP_BITS - 1)); - } - - static void setTabAt(Node[] tab, int i, Node v) { - Unsafe.getUnsafe().putObjectVolatile(tab, ((long) i << ASHIFT) + ABASE, v); - } - - /** - * Spreads (XORs) higher bits of hash to lower and also forces top - * bit to 0. Because the table uses power-of-two masking, sets of - * hashes that vary only in bits above the current mask will - * always collide. (Among known examples are sets of Float keys - * holding consecutive whole numbers in small tables.) So we - * apply a transform that spreads the impact of higher bits - * downward. There is a tradeoff between speed, utility, and - * quality of bit-spreading. Because many common sets of hashes - * are already reasonably distributed (so don't benefit from - * spreading), and because we use trees to handle large sets of - * collisions in bins, we just XOR some shifted bits in the - * cheapest possible way to reduce systematic lossage, as well as - * to incorporate impact of the highest bits that would otherwise - * never be used in index calculations because of table bounds. - */ - static int spread(int h) { - return (h ^ (h >>> 16)) & HASH_BITS; - } - - @SuppressWarnings("unchecked") - static Node tabAt(Node[] tab, int i) { - return (Node) Unsafe.getUnsafe().getObjectVolatile(tab, ((long) i << ASHIFT) + ABASE); - } - - /** - * Returns a list on non-TreeNodes replacing those in given list. - */ - static Node untreeify(Node b) { - Node hd = null, tl = null; - for (Node q = b; q != null; q = q.next) { - Node p = new Node<>(q.hash, q.key, q.val, null); - if (tl == null) - hd = p; - else - tl.next = p; - tl = p; - } - return hd; - } - - /** - * Helps transfer if a resize is in progress. - */ - final Node[] helpTransfer(Node[] tab, Node f) { - Node[] nextTab; - int sc; - if (tab != null && (f instanceof ForwardingNode) && - (nextTab = ((ForwardingNode) f).nextTable) != null) { - int rs = resizeStamp(tab.length); - while (nextTab == nextTable && table == tab && - (sc = sizeCtl) < 0) { - if ((sc >>> RESIZE_STAMP_SHIFT) != rs || sc == rs + 1 || - sc == rs + MAX_RESIZERS || transferIndex <= 0) - break; - if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc, sc + 1)) { - transfer(tab, nextTab); - break; - } - } - return nextTab; - } - return table; - } - - /* ----------------Views -------------- */ - - /** - * Implementation for put and putIfAbsent - */ - final V putVal(int key, V value, boolean onlyIfAbsent) { - if (key < 0) throw new IllegalArgumentException(); - if (value == null) throw new NullPointerException(); - int hash = spread(keyHashCode(key)); - int binCount = 0; - Node _new = null; - - for (Node[] tab = table; ; ) { - Node f; - int n, i, fh; - if (tab == null || (n = tab.length) == 0) - tab = initTable(); - else if ((f = tabAt(tab, i = (n - 1) & hash)) == null) { - if (_new == null) { - _new = new Node<>(hash, key, value, null); - } - if (casTabAt(tab, i, _new)) { - break; // no lock when adding to empty bin - } - } else if ((fh = f.hash) == MOVED) - tab = helpTransfer(tab, f); - else { - V oldVal = null; - synchronized (f) { - if (tabAt(tab, i) == f) { - if (fh >= 0) { - binCount = 1; - for (Node e = f; ; ++binCount) { - if (e.hash == hash && e.key == key) { - oldVal = e.val; - if (!onlyIfAbsent) - e.val = value; - break; - } - Node pred = e; - if ((e = e.next) == null) { - if (_new == null) { - pred.next = new Node<>(hash, key, value, null); - } else { - pred.next = _new; - } - break; - } - } - } else if (f instanceof TreeBin) { - Node p; - binCount = 2; - if ((p = ((TreeBin) f).putTreeVal(hash, key, value)) != null) { - oldVal = p.val; - if (!onlyIfAbsent) - p.val = value; - } - } - } - } - if (binCount != 0) { - if (binCount >= TREEIFY_THRESHOLD) - treeifyBin(tab, i); - if (oldVal != null) - return oldVal; - break; - } - } - } - addCount(1L, binCount); - return null; - } - - /** - * Implementation for the four public remove/replace methods: - * Replaces node value with v, conditional upon match of cv if - * non-null. If resulting value is null, delete. - */ - final V replaceNode(int key, V value, V cv) { - int hash = spread(keyHashCode(key)); - for (Node[] tab = table; ; ) { - Node f; - int n, i, fh; - if (tab == null || (n = tab.length) == 0 || - (f = tabAt(tab, i = (n - 1) & hash)) == null) - break; - else if ((fh = f.hash) == MOVED) - tab = helpTransfer(tab, f); - else { - V oldVal = null; - boolean validated = false; - synchronized (f) { - if (tabAt(tab, i) == f) { - if (fh >= 0) { - validated = true; - for (Node e = f, pred = null; ; ) { - if (e.hash == hash && e.key == key) { - V ev = e.val; - if (cv == null || cv == ev || (cv.equals(ev))) { - oldVal = ev; - if (value != null) - e.val = value; - else if (pred != null) - pred.next = e.next; - else - setTabAt(tab, i, e.next); - } - break; - } - pred = e; - if ((e = e.next) == null) - break; - } - } else if (f instanceof TreeBin) { - validated = true; - TreeBin t = (TreeBin) f; - TreeNode r, p; - if ((r = t.root) != null && - (p = r.findTreeNode(hash, key)) != null) { - V pv = p.val; - if (cv == null || cv == pv || cv.equals(pv)) { - oldVal = pv; - if (value != null) - p.val = value; - else if (t.removeTreeNode(p)) - setTabAt(tab, i, untreeify(t.first)); - } - } - } - } - } - if (validated) { - if (oldVal != null) { - if (value == null) - addCount(-1L, -1); - return oldVal; - } - break; - } - } - } - return null; - } - - final long sumCount() { - CounterCell[] as = counterCells; - CounterCell a; - long sum = baseCount; - if (as != null) { - for (int i = 0; i < as.length; ++i) { - if ((a = as[i]) != null) - sum += a.value; - } - } - return sum; - } - - public interface IntEntry { - boolean equals(Object var1); - - int getKey(); - - V getValue(); - - int hashCode(); - - V setValue(V var1); - } - - /** - * Base of key, value, and entry Iterators. Adds fields to - * Traverser to support iterator.remove. - */ - static class BaseIterator extends Traverser { - Node lastReturned; - ConcurrentIntHashMap map; - - public final boolean hasNext() { - return next != null; - } - - public final void remove() { - Node p; - if ((p = lastReturned) == null) - throw new IllegalStateException(); - lastReturned = null; - map.replaceNode(p.key, null, null); - } - - void of(ConcurrentIntHashMap map) { - Node[] tab = map.table; - int l = tab == null ? 0 : tab.length; - super.of(tab, l, l); - this.map = map; - advance(); - } - } - - /** - * Base class for views. - */ - abstract static class CollectionView - implements Collection, Serializable { - private static final String oomeMsg = "Required array size too large"; - private static final long serialVersionUID = 7249069246763182397L; - final ConcurrentIntHashMap map; - - CollectionView(ConcurrentIntHashMap map) { - this.map = map; - } - - /** - * Removes all of the elements from this view, by removing all - * the mappings from the map backing this view. - */ - public final void clear() { - map.clear(); - } - - public abstract boolean contains(Object o); - - public final boolean containsAll(@NotNull Collection c) { - if (c != this) { - for (Object e : c) { - if (e == null || !contains(e)) - return false; - } - } - return true; - } - - /** - * Returns the map backing this view. - * - * @return the map backing this view - */ - public ConcurrentIntHashMap getMap() { - return map; - } - - public final boolean isEmpty() { - return map.isEmpty(); - } - - /** - * Returns an iterator over the elements in this collection. - *

    The returned iterator is - * weakly consistent. - * - * @return an iterator over the elements in this collection - */ - @NotNull - public abstract Iterator iterator(); - - public abstract boolean remove(Object o); - - @Override - public final boolean removeAll(@NotNull Collection c) { - boolean modified = false; - for (Iterator it = iterator(); it.hasNext(); ) { - if (c.contains(it.next())) { - it.remove(); - modified = true; - } - } - return modified; - } - - // implementations below rely on concrete classes supplying these - // abstract methods - - @Override - public final boolean retainAll(@NotNull Collection c) { - boolean modified = false; - for (Iterator it = iterator(); it.hasNext(); ) { - if (!c.contains(it.next())) { - it.remove(); - modified = true; - } - } - return modified; - } - - @Override - public final int size() { - return map.size(); - } - - @Override - public final Object @NotNull [] toArray() { - long sz = map.mappingCount(); - if (sz > MAX_ARRAY_SIZE) - throw new OutOfMemoryError(oomeMsg); - int n = (int) sz; - Object[] r = new Object[n]; - int i = 0; - for (E e : this) { - if (i == n) { - if (n >= MAX_ARRAY_SIZE) - throw new OutOfMemoryError(oomeMsg); - if (n >= MAX_ARRAY_SIZE - (MAX_ARRAY_SIZE >>> 1) - 1) - n = MAX_ARRAY_SIZE; - else - n += (n >>> 1) + 1; - r = Arrays.copyOf(r, n); - } - r[i++] = e; - } - return (i == n) ? r : Arrays.copyOf(r, i); - } - - @Override - @SuppressWarnings("unchecked") - public final T @NotNull [] toArray(@NotNull T @NotNull [] a) { - long sz = map.mappingCount(); - if (sz > MAX_ARRAY_SIZE) - throw new OutOfMemoryError(oomeMsg); - int m = (int) sz; - T[] r = (a.length >= m) ? a : - (T[]) java.lang.reflect.Array - .newInstance(a.getClass().getComponentType(), m); - int n = r.length; - int i = 0; - for (E e : this) { - if (i == n) { - if (n >= MAX_ARRAY_SIZE) - throw new OutOfMemoryError(oomeMsg); - if (n >= MAX_ARRAY_SIZE - (MAX_ARRAY_SIZE >>> 1) - 1) - n = MAX_ARRAY_SIZE; - else - n += (n >>> 1) + 1; - r = Arrays.copyOf(r, n); - } - r[i++] = (T) e; - } - if (a == r && i < n) { - r[i] = null; // null-terminate - return r; - } - return (i == n) ? r : Arrays.copyOf(r, i); - } - - /** - * Returns a string representation of this collection. - * The string representation consists of the string representations - * of the collection's elements in the order they are returned by - * its iterator, enclosed in square brackets ({@code "[]"}). - * Adjacent elements are separated by the characters {@code ", "} - * (comma and space). Elements are converted to strings as by - * {@link String#valueOf(Object)}. - * - * @return a string representation of this collection - */ - @Override - public final String toString() { - StringBuilder sb = new StringBuilder(); - sb.append('['); - Iterator it = iterator(); - if (it.hasNext()) { - for (; ; ) { - Object e = it.next(); - sb.append(e == this ? "(this Collection)" : e); - if (!it.hasNext()) - break; - sb.append(',').append(' '); - } - } - return sb.append(']').toString(); - } - } - - /** - * A padded cell for distributing counts. Adapted from LongAdder - * and Striped64. See their internal docs for explanation. - */ - static final class CounterCell { - final long value; - - CounterCell(long x) { - value = x; - } - } - - static final class EntryIterator extends BaseIterator - implements Iterator> { - - @Override - public IntEntry next() { - Node p; - if ((p = next) == null) - throw new NoSuchElementException(); - int k = p.key; - V v = p.val; - lastReturned = p; - advance(); - return new MapEntry<>(k, v, map); - } - } - - /** - * A view of a ConcurrentLongHashMap as a {@link Set} of (key, value) - * entries. This class cannot be directly instantiated. See - * {@link #entrySet()}. - */ - static final class EntrySetView extends CollectionView> - implements Set>, Serializable { - private static final long serialVersionUID = 2249069246763182397L; - - private final ThreadLocal> tlEntryIterator = ThreadLocal.withInitial(EntryIterator::new); - - EntrySetView(ConcurrentIntHashMap map) { - super(map); - } - - @Override - public boolean add(IntEntry e) { - return map.putVal(e.getKey(), e.getValue(), false) == null; - } - - @Override - public boolean addAll(@NotNull Collection> c) { - boolean added = false; - for (IntEntry e : c) { - if (add(e)) - added = true; - } - return added; - } - - @Override - public boolean contains(Object o) { - int k; - Object v, r; - IntEntry e; - return ((o instanceof IntEntry) && - (k = (e = (IntEntry) o).getKey()) != EMPTY_KEY && - (r = map.get(k)) != null && - (v = e.getValue()) != null && - (v == r || v.equals(r))); - } - - @Override - public boolean equals(Object o) { - Set c; - return ((o instanceof Set) && - ((c = (Set) o) == this || - (containsAll(c) && c.containsAll(this)))); - } - - @Override - public int hashCode() { - int h = 0; - Node[] t = map.table; - if (t != null) { - Traverser it = map.getTraverser(t); - for (Node p; (p = it.advance()) != null; ) { - h += p.hashCode(); - } - } - return h; - } - - /** - * @return an iterator over the entries of the backing map - */ - @NotNull - public Iterator> iterator() { - EntryIterator it = tlEntryIterator.get(); - it.of(map); - return it; - } - - @Override - public boolean remove(Object o) { - int k; - Object v; - IntEntry e; - return ((o instanceof IntEntry) && - (k = (e = (IntEntry) o).getKey()) != EMPTY_KEY && - (v = e.getValue()) != null && - map.remove(k, (V) v)); - } - } - - /** - * A node inserted at head of bins during transfer operations. - */ - static final class ForwardingNode extends Node { - final Node[] nextTable; - - ForwardingNode(Node[] tab) { - super(MOVED, EMPTY_KEY, null, null); - this.nextTable = tab; - } - - @Override - Node find(int h, int k) { - // loop to avoid arbitrarily deep recursion on forwarding nodes - outer: - for (Node[] tab = nextTable; ; ) { - Node e; - int n; - if (k == EMPTY_KEY || tab == null || (n = tab.length) == 0 || - (e = tabAt(tab, (n - 1) & h)) == null) - return null; - for (; ; ) { - int eh; - if ((eh = e.hash) == h && e.key == k) - return e; - if (eh < 0) { - if (e instanceof ForwardingNode) { - tab = ((ForwardingNode) e).nextTable; - continue outer; - } else - return e.find(h, k); - } - if ((e = e.next) == null) - return null; - } - } - } - } - - public static final class KeyIterator extends BaseIterator { - - public int next() { - Node p; - if ((p = next) == null) - throw new NoSuchElementException(); - int k = p.key; - lastReturned = p; - advance(); - return k; - } - } - - /** - * A view of a ConcurrentLongHashMap as a long set of keys, in - * which additions may optionally be enabled by mapping to a - * common value. This class cannot be directly instantiated. - * See {@link #keySet() keySet()}, - * {@link #keySet(Object) keySet(V)}, - * {@link #newKeySet() newKeySet()}, - * {@link #newKeySet(int) newKeySet(int)}. - * - * @since 1.8 - */ - public static class KeySetView implements Serializable { - private static final long serialVersionUID = 7249069246763182397L; - private final ConcurrentIntHashMap map; - private final ThreadLocal> tlKeyIterator = ThreadLocal.withInitial(KeyIterator::new); - private final V value; - - KeySetView(ConcurrentIntHashMap map, V value) { // non-public - this.map = map; - this.value = value; - } - - /** - * Adds the specified key to this set view by mapping the key to - * the default mapped value in the backing map, if defined. - * - * @param k key to be added - * @return {@code true} if this set changed as a result of the call - * @throws NullPointerException if the specified key is null - * @throws UnsupportedOperationException if no default mapped value - * for additions was provided - */ - public boolean add(int k) { - V v; - if ((v = value) == null) - throw new UnsupportedOperationException(); - return map.putVal(k, v, true) == null; - } - - public final void clear() { - map.clear(); - } - - public boolean contains(int k) { - return map.containsKey(k); - } - - @Override - public boolean equals(Object o) { - KeySetView c; - return ((o instanceof KeySetView) && - ((c = (KeySetView) o) == this || - (containsAll(c) && c.containsAll(this)))); - } - - /** - * Returns the default mapped value for additions, - * or {@code null} if additions are not supported. - * - * @return the default mapped value for additions, or {@code null} - * if not supported - */ - public V getMappedValue() { - return value; - } - - @Override - public int hashCode() { - int h = 0; - KeyIterator it = iterator(); - if (it.hasNext()) { - do { - int k = it.next(); - h += keyHashCode(k); - } while (it.hasNext()); - } - return h; - } - - public final boolean isEmpty() { - return map.isEmpty(); - } - - /** - * @return an iterator over the keys of the backing map - */ - @NotNull - public KeyIterator iterator() { - KeyIterator it = tlKeyIterator.get(); - it.of(map); - return it; - } - - /** - * Removes the key from this map view, by removing the key (and its - * corresponding value) from the backing map. This method does - * nothing if the key is not in the map. - * - * @param k the key to be removed from the backing map - * @return {@code true} if the backing map contained the specified key - * @throws NullPointerException if the specified key is null - */ - public boolean remove(int k) { - return map.remove(k) != null; - } - - public final int size() { - return map.size(); - } - - @Override - public final String toString() { - StringBuilder sb = new StringBuilder(); - sb.append('['); - KeyIterator it = iterator(); - if (it.hasNext()) { - for (; ; ) { - int k = it.next(); - sb.append(k); - if (!it.hasNext()) - break; - sb.append(',').append(' '); - } - } - return sb.append(']').toString(); - } - - private boolean containsAll(@NotNull KeySetView c) { - KeyIterator it = iterator(); - if (it.hasNext()) { - do { - int k = it.next(); - if (!contains(k)) - return false; - } while (it.hasNext()); - } - return true; - } - } - - /** - * Exported Entry for EntryIterator - */ - static final class MapEntry implements IntEntry { - final int key; // != EMPTY_KEY - final ConcurrentIntHashMap map; - V val; // non-null - - MapEntry(int key, V val, ConcurrentIntHashMap map) { - this.key = key; - this.val = val; - this.map = map; - } - - @Override - public boolean equals(Object o) { - int k; - Object v; - IntEntry e; - return ((o instanceof IntEntry) && - (k = (e = (IntEntry) o).getKey()) != EMPTY_KEY && - (v = e.getValue()) != null && - (k == key) && - (v == val || v.equals(val))); - } - - @Override - public int getKey() { - return key; - } - - @Override - public V getValue() { - return val; - } - - @Override - public int hashCode() { - return keyHashCode(key) ^ val.hashCode(); - } - - /** - * Sets our entry's value and writes through to the map. The - * value to return is somewhat arbitrary here. Since we do not - * necessarily track asynchronous changes, the most recent - * "previous" value could be different from what we return (or - * could even have been removed, in which case the put will - * re-establish). We do not and cannot guarantee more. - */ - @NotNull - public V setValue(V value) { - if (value == null) throw new NullPointerException(); - V v = val; - val = value; - map.put(key, value); - return v; - } - - @Override - public String toString() { - return key + "=" + val; - } - } - - /** - * Key-value entry. This class is never exported out as a - * user-mutable Map.Entry (i.e., one supporting setValue; see - * MapEntry below), but can be used for read-only traversals used - * in bulk tasks. Subclasses of Node with a negative hash field - * are special, and contain null keys and values (but are never - * exported). Otherwise, keys and vals are never null. - */ - static class Node implements IntEntry { - final int hash; - final int key; - volatile Node next; - volatile V val; - - Node(int hash, int key, V val, Node next) { - this.hash = hash; - this.key = key; - this.val = val; - this.next = next; - } - - @Override - public final boolean equals(Object o) { - int k; - Object v, u; - IntEntry e; - return ((o instanceof IntEntry) && - (k = (e = (IntEntry) o).getKey()) != EMPTY_KEY && - (v = e.getValue()) != null && - (k == key) && - (v == (u = val) || v.equals(u))); - } - - @Override - public final int getKey() { - return key; - } - - @Override - public final V getValue() { - return val; - } - - @Override - public final int hashCode() { - return keyHashCode(key) ^ val.hashCode(); - } - - @Override - public final V setValue(V value) { - throw new UnsupportedOperationException(); - } - - @Override - public final String toString() { - return key + "=" + val; - } - - /** - * Virtualized support for map.get(); overridden in subclasses. - */ - Node find(int h, int k) { - Node e = this; - if (k != EMPTY_KEY) { - do { - if (e.hash == h && (e.key == k)) - return e; - } while ((e = e.next) != null); - } - return null; - } - } - - /** - * A place-holder node used in computeIfAbsent and compute - */ - static final class ReservationNode extends Node { - ReservationNode() { - super(RESERVED, EMPTY_KEY, null, null); - } - - @Override - Node find(int h, int k) { - return null; - } - } - - /** - * Stripped-down version of helper class used in previous version, - * declared for the sake of serialization compatibility - */ - static class Segment extends ReentrantLock implements Serializable { - private static final long serialVersionUID = 2249069246763182397L; - final float loadFactor; - - Segment() { - this.loadFactor = ConcurrentIntHashMap.LOAD_FACTOR; - } - } - - /** - * Records the table, its length, and current traversal index for a - * traverser that must process a region of a forwarded table before - * proceeding with current table. - */ - static final class TableStack { - int index; - int length; - TableStack next; - Node[] tab; - } - - /** - * Encapsulates traversal for methods such as containsValue; also - * serves as a base class for other iterators and spliterators. - *

    - * Method advance visits once each still-valid node that was - * reachable upon iterator construction. It might miss some that - * were added to a bin after the bin was visited, which is OK wrt - * consistency guarantees. Maintaining this property in the face - * of possible ongoing resizes requires a fair amount of - * bookkeeping state that is difficult to optimize away amidst - * volatile accesses. Even so, traversal maintains reasonable - * throughput. - *

    - * Normally, iteration proceeds bin-by-bin traversing lists. - * However, if the table has been resized, then all future steps - * must traverse both the bin at the current index as well as at - * (index + baseSize); and so on for further resizings. To - * paranoically cope with potential sharing by users of iterators - * across threads, iteration terminates if a bounds checks fails - * for a table read. - */ - static class Traverser { - int baseIndex; // current index of initial table - int baseLimit; // index bound for initial table - int baseSize; // initial table size - int index; // index of bin to use next - Node next; // the next entry to use - TableStack stack, spare; // to save/restore on ForwardingNodes - Node[] tab; // current table; updated if resized - - /** - * Saves traversal state upon encountering a forwarding node. - */ - private void pushState(Node[] t, int i, int n) { - TableStack s = spare; // reuse if possible - if (s != null) - spare = s.next; - else - s = new TableStack<>(); - s.tab = t; - s.length = n; - s.index = i; - s.next = stack; - stack = s; - } - - /** - * Possibly pops traversal state. - * - * @param n length of current table - */ - private void recoverState(int n) { - TableStack s; - int len; - while ((s = stack) != null && (index += (len = s.length)) >= n) { - n = len; - index = s.index; - tab = s.tab; - s.tab = null; - TableStack next = s.next; - s.next = spare; // save for reuse - stack = next; - spare = s; - } - if (s == null && (index += baseSize) >= n) - index = ++baseIndex; - } - - /** - * Advances if possible, returning next valid node, or null if none. - */ - final Node advance() { - Node e; - if ((e = next) != null) - e = e.next; - for (; ; ) { - Node[] t; - int i, n; // must use locals in checks - if (e != null) - return next = e; - if (baseIndex >= baseLimit || (t = tab) == null || - (n = t.length) <= (i = index) || i < 0) - return next = null; - if ((e = tabAt(t, i)) != null && e.hash < 0) { - if (e instanceof ForwardingNode) { - tab = ((ForwardingNode) e).nextTable; - e = null; - pushState(t, i, n); - continue; - } else if (e instanceof TreeBin) - e = ((TreeBin) e).first; - else - e = null; - } - if (stack != null) - recoverState(n); - else if ((index = i + baseSize) >= n) - index = ++baseIndex; // visit upper slots if present - } - } - - void of(Node[] tab, int size, int limit) { - this.tab = tab; - this.baseSize = size; - this.baseIndex = this.index = 0; - this.baseLimit = limit; - this.next = null; - } - } - - /** - * TreeNodes used at the heads of bins. TreeBins do not hold user - * keys or values, but instead point to list of TreeNodes and - * their root. They also maintain a parasitic read-write lock - * forcing writers (who hold bin lock) to wait for readers (who do - * not) to complete before tree restructuring operations. - */ - static final class TreeBin extends Node { - static final int READER = 4; // increment value for setting read lock - static final int WAITER = 2; // set when waiting for write lock - // values for lockState - static final int WRITER = 1; // set while holding write lock - private static final long LOCKSTATE; - private static final sun.misc.Unsafe U; - volatile TreeNode first; - volatile int lockState; - TreeNode root; - volatile Thread waiter; - - /** - * Creates bin with initial set of nodes headed by b. - */ - TreeBin(TreeNode b) { - super(TREEBIN, EMPTY_KEY, null, null); - this.first = b; - TreeNode r = null; - for (TreeNode x = b, next; x != null; x = next) { - next = (TreeNode) x.next; - x.left = x.right = null; - if (r == null) { - x.parent = null; - x.red = false; - r = x; - } else { - int k = x.key; - int h = x.hash; - for (TreeNode p = r; ; ) { - int dir, ph; - int pk = p.key; - if ((ph = p.hash) > h) - dir = -1; - else if (ph < h) - dir = 1; - else if ((dir = compareComparables(k, pk)) == 0) - dir = tieBreakOrder(k, pk); - TreeNode xp = p; - if ((p = (dir <= 0) ? p.left : p.right) == null) { - x.parent = xp; - if (dir <= 0) - xp.left = x; - else - xp.right = x; - r = balanceInsertion(r, x); - break; - } - } - } - } - this.root = r; - assert checkInvariants(root); - } - - /** - * Possibly blocks awaiting root lock. - */ - private void contendedLock() { - boolean waiting = false; - for (int s; ; ) { - if (((s = lockState) & ~WAITER) == 0) { - if (U.compareAndSwapInt(this, LOCKSTATE, s, WRITER)) { - if (waiting) - waiter = null; - return; - } - } else if ((s & WAITER) == 0) { - if (U.compareAndSwapInt(this, LOCKSTATE, s, s | WAITER)) { - waiting = true; - waiter = Thread.currentThread(); - } - } else if (waiting) - LockSupport.park(this); - } - } - - /** - * Acquires write lock for tree restructuring. - */ - private void lockRoot() { - if (!U.compareAndSwapInt(this, LOCKSTATE, 0, WRITER)) - contendedLock(); // offload to separate method - } - - /** - * Releases write lock for tree restructuring. - */ - private void unlockRoot() { - lockState = 0; - } - - static TreeNode balanceDeletion(TreeNode root, - TreeNode x) { - for (TreeNode xp, xpl, xpr; ; ) { - if (x == null || x == root) - return root; - else if ((xp = x.parent) == null) { - x.red = false; - return x; - } else if (x.red) { - x.red = false; - return root; - } else if ((xpl = xp.left) == x) { - if ((xpr = xp.right) != null && xpr.red) { - xpr.red = false; - xp.red = true; - root = rotateLeft(root, xp); - xpr = (xp = x.parent) == null ? null : xp.right; - } - if (xpr == null) - x = xp; - else { - TreeNode sl = xpr.left, sr = xpr.right; - if ((sr == null || !sr.red) && - (sl == null || !sl.red)) { - xpr.red = true; - x = xp; - } else { - if (sr == null || !sr.red) { - sl.red = false; - xpr.red = true; - root = rotateRight(root, xpr); - xpr = (xp = x.parent) == null ? - null : xp.right; - } - if (xpr != null) { - xpr.red = xp.red; - if ((sr = xpr.right) != null) - sr.red = false; - } - if (xp != null) { - xp.red = false; - root = rotateLeft(root, xp); - } - x = root; - } - } - } else { // symmetric - if (xpl != null && xpl.red) { - xpl.red = false; - xp.red = true; - root = rotateRight(root, xp); - xpl = (xp = x.parent) == null ? null : xp.left; - } - if (xpl == null) - x = xp; - else { - TreeNode sl = xpl.left, sr = xpl.right; - if ((sl == null || !sl.red) && - (sr == null || !sr.red)) { - xpl.red = true; - x = xp; - } else { - if (sl == null || !sl.red) { - sr.red = false; - xpl.red = true; - root = rotateLeft(root, xpl); - xpl = (xp = x.parent) == null ? - null : xp.left; - } - if (xpl != null) { - xpl.red = xp.red; - if ((sl = xpl.left) != null) - sl.red = false; - } - if (xp != null) { - xp.red = false; - root = rotateRight(root, xp); - } - x = root; - } - } - } - } - } - - static TreeNode balanceInsertion(TreeNode root, - TreeNode x) { - x.red = true; - for (TreeNode xp, xpp, xppl, xppr; ; ) { - if ((xp = x.parent) == null) { - x.red = false; - return x; - } else if (!xp.red || (xpp = xp.parent) == null) - return root; - if (xp == (xppl = xpp.left)) { - if ((xppr = xpp.right) != null && xppr.red) { - xppr.red = false; - xp.red = false; - xpp.red = true; - x = xpp; - } else { - if (x == xp.right) { - root = rotateLeft(root, x = xp); - xpp = (xp = x.parent) == null ? null : xp.parent; - } - if (xp != null) { - xp.red = false; - if (xpp != null) { - xpp.red = true; - root = rotateRight(root, xpp); - } - } - } - } else { - if (xppl != null && xppl.red) { - xppl.red = false; - xp.red = false; - xpp.red = true; - x = xpp; - } else { - if (x == xp.left) { - root = rotateRight(root, x = xp); - xpp = (xp = x.parent) == null ? null : xp.parent; - } - if (xp != null) { - xp.red = false; - if (xpp != null) { - xpp.red = true; - root = rotateLeft(root, xpp); - } - } - } - } - } - } - - /** - * Recursive invariant check - */ - @SuppressWarnings("SimplifiableIfStatement") - static boolean checkInvariants(TreeNode t) { - TreeNode tp = t.parent, tl = t.left, tr = t.right, - tb = t.prev, tn = (TreeNode) t.next; - if (tb != null && tb.next != t) - return false; - if (tn != null && tn.prev != t) - return false; - if (tp != null && t != tp.left && t != tp.right) - return false; - if (tl != null && (tl.parent != t || tl.hash > t.hash)) - return false; - if (tr != null && (tr.parent != t || tr.hash < t.hash)) - return false; - if (t.red && tl != null && tl.red && tr != null && tr.red) - return false; - if (tl != null && !checkInvariants(tl)) - return false; - return !(tr != null && !checkInvariants(tr)); - } - - static TreeNode rotateLeft(TreeNode root, TreeNode p) { - TreeNode r, pp, rl; - if (p != null && (r = p.right) != null) { - if ((rl = p.right = r.left) != null) - rl.parent = p; - if ((pp = r.parent = p.parent) == null) - (root = r).red = false; - else if (pp.left == p) - pp.left = r; - else - pp.right = r; - r.left = p; - p.parent = r; - } - return root; - } - - static TreeNode rotateRight(TreeNode root, TreeNode p) { - TreeNode l, pp, lr; - if (p != null && (l = p.left) != null) { - if ((lr = p.left = l.right) != null) - lr.parent = p; - if ((pp = l.parent = p.parent) == null) - (root = l).red = false; - else if (pp.right == p) - pp.right = l; - else - pp.left = l; - l.right = p; - p.parent = l; - } - return root; - } - - /** - * Tie-breaking utility for ordering insertions when equal - * hashCodes and non-comparable. We don't require a total - * order, just a consistent insertion rule to maintain - * equivalence across rebalancings. Tie-breaking further than - * necessary simplifies testing a bit. - */ - static int tieBreakOrder(Object a, Object b) { - int d; - if (a == null || b == null || - (d = a.getClass().getName(). - compareTo(b.getClass().getName())) == 0) - d = (System.identityHashCode(a) <= System.identityHashCode(b) ? - -1 : 1); - return d; - } - - /** - * Returns matching node or null if none. Tries to search - * using tree comparisons from root, but continues linear - * search when lock not available. - */ - @Override - Node find(int h, int k) { - if (k != EMPTY_KEY) { - for (Node e = first; e != null; ) { - int s; - if (((s = lockState) & (WAITER | WRITER)) != 0) { - if (e.hash == h && e.key == k) - return e; - e = e.next; - } else if (U.compareAndSwapInt(this, LOCKSTATE, s, - s + READER)) { - TreeNode r, p; - try { - p = ((r = root) == null ? null : - r.findTreeNode(h, k)); - } finally { - Thread w; - if (U.getAndAddInt(this, LOCKSTATE, -READER) == - (READER | WAITER) && (w = waiter) != null) - LockSupport.unpark(w); - } - return p; - } - } - } - return null; - } - - /** - * Finds or adds a node. - * - * @return null if added - */ - TreeNode putTreeVal(int h, int k, V v) { - boolean searched = false; - for (TreeNode p = root; ; ) { - int dir, ph; - int pk; - if (p == null) { - first = root = new TreeNode<>(h, k, v, null, null); - break; - } else if ((ph = p.hash) > h) - dir = -1; - else if (ph < h) - dir = 1; - else if ((pk = p.key) == k) - return p; - else if ((dir = compareComparables(k, pk)) == 0) { - if (!searched) { - TreeNode q, ch; - searched = true; - if (((ch = p.left) != null && - (q = ch.findTreeNode(h, k)) != null) || - ((ch = p.right) != null && - (q = ch.findTreeNode(h, k)) != null)) - return q; - } - dir = tieBreakOrder(k, pk); - } - - TreeNode xp = p; - if ((p = (dir <= 0) ? p.left : p.right) == null) { - TreeNode x, f = first; - first = x = new TreeNode<>(h, k, v, f, xp); - if (f != null) - f.prev = x; - if (dir <= 0) - xp.left = x; - else - xp.right = x; - if (!xp.red) - x.red = true; - else { - lockRoot(); - try { - root = balanceInsertion(root, x); - } finally { - unlockRoot(); - } - } - break; - } - } - assert checkInvariants(root); - return null; - } - - /** - * Removes the given node, that must be present before this - * call. This is messier than typical red-black deletion code - * because we cannot swap the contents of an interior node - * with a leaf successor that is pinned by "next" pointers - * that are accessible independently of lock. So instead we - * swap the tree linkages. - * - * @return true if now too small, so should be untreeified - */ - boolean removeTreeNode(TreeNode p) { - TreeNode next = (TreeNode) p.next; - TreeNode pred = p.prev; // unlink traversal pointers - TreeNode r, rl; - if (pred == null) - first = next; - else - pred.next = next; - if (next != null) - next.prev = pred; - if (first == null) { - root = null; - return true; - } - if ((r = root) == null || r.right == null || // too small - (rl = r.left) == null || rl.left == null) - return true; - lockRoot(); - try { - TreeNode replacement; - TreeNode pl = p.left; - TreeNode pr = p.right; - if (pl != null && pr != null) { - TreeNode s = pr, sl; - while ((sl = s.left) != null) // find successor - s = sl; - boolean c = s.red; - s.red = p.red; - p.red = c; // swap colors - TreeNode sr = s.right; - TreeNode pp = p.parent; - if (s == pr) { // p was s's direct parent - p.parent = s; - s.right = p; - } else { - TreeNode sp = s.parent; - if ((p.parent = sp) != null) { - if (s == sp.left) - sp.left = p; - else - sp.right = p; - } - s.right = pr; - pr.parent = s; - } - p.left = null; - if ((p.right = sr) != null) - sr.parent = p; - s.left = pl; - pl.parent = s; - if ((s.parent = pp) == null) - r = s; - else if (p == pp.left) - pp.left = s; - else - pp.right = s; - if (sr != null) - replacement = sr; - else - replacement = p; - } else if (pl != null) - replacement = pl; - else if (pr != null) - replacement = pr; - else - replacement = p; - if (replacement != p) { - TreeNode pp = replacement.parent = p.parent; - if (pp == null) - r = replacement; - else if (p == pp.left) - pp.left = replacement; - else - pp.right = replacement; - p.left = p.right = p.parent = null; - } - - root = (p.red) ? r : balanceDeletion(r, replacement); - - if (p == replacement) { // detach pointers - TreeNode pp; - if ((pp = p.parent) != null) { - if (p == pp.left) - pp.left = null; - else if (p == pp.right) - pp.right = null; - p.parent = null; - } - } - } finally { - unlockRoot(); - } - assert checkInvariants(root); - return false; - } - - static { - try { - U = Unsafe.getUnsafe(); - Class k = TreeBin.class; - LOCKSTATE = U.objectFieldOffset - (k.getDeclaredField("lockState")); - } catch (Exception e) { - throw new Error(e); - } - } - - - - - - - - - - - - - - - - /* ------------------------------------------------------------ */ - // Red-black tree methods, all adapted from CLR - } - - /** - * Nodes for use in TreeBins - */ - static final class TreeNode extends Node { - TreeNode left; - TreeNode parent; // red-black tree links - TreeNode prev; // needed to unlink next upon deletion - boolean red; - TreeNode right; - - TreeNode(int hash, int key, V val, Node next, TreeNode parent) { - super(hash, key, val, next); - this.parent = parent; - } - - @Override - Node find(int h, int k) { - return findTreeNode(h, k); - } - - /** - * Returns the TreeNode (or null if not found) for the given key - * starting at given root. - */ - TreeNode findTreeNode(int h, int k) { - if (k != EMPTY_KEY) { - TreeNode p = this; - do { - int ph, dir; - int pk; - TreeNode q; - TreeNode pl = p.left, pr = p.right; - if ((ph = p.hash) > h) - p = pl; - else if (ph < h) - p = pr; - else if ((pk = p.key) == k) - return p; - else if (pl == null) - p = pr; - else if (pr == null) - p = pl; - else if ((dir = compareComparables(k, pk)) != 0) - p = (dir < 0) ? pl : pr; - else if ((q = pr.findTreeNode(h, k)) != null) - return q; - else - p = pl; - } while (p != null); - } - return null; - } - - - } - - static final class ValueIterator extends BaseIterator - implements Iterator { - public V next() { - Node p; - if ((p = next) == null) - throw new NoSuchElementException(); - V v = p.val; - lastReturned = p; - advance(); - return v; - } - } - - /** - * A view of a ConcurrentLongHashMap as a {@link Collection} of - * values, in which additions are disabled. This class cannot be - * directly instantiated. See {@link #values()}. - */ - static final class ValuesView extends CollectionView - implements Collection, Serializable { - private static final long serialVersionUID = 2249069246763182397L; - private final ThreadLocal> tlValueIterator = ThreadLocal.withInitial(ValueIterator::new); - - ValuesView(ConcurrentIntHashMap map) { - super(map); - } - - @Override - public boolean add(V e) { - throw new UnsupportedOperationException(); - } - - @Override - public boolean addAll(@NotNull Collection c) { - throw new UnsupportedOperationException(); - } - - @Override - public boolean contains(Object o) { - return map.containsValue((V) o); - } - - @Override - @NotNull - public Iterator iterator() { - ValueIterator it = tlValueIterator.get(); - it.of(map); - return it; - } - - @Override - public boolean remove(Object o) { - if (o != null) { - for (Iterator it = iterator(); it.hasNext(); ) { - if (o.equals(it.next())) { - it.remove(); - return true; - } - } - } - return false; - } - } - - static { - try { - Class tk = Thread.class; - SEED = Unsafe.getUnsafe().objectFieldOffset(tk.getDeclaredField("threadLocalRandomSeed")); - PROBE = Unsafe.getUnsafe().objectFieldOffset(tk.getDeclaredField("threadLocalRandomProbe")); - } catch (Exception e) { - throw new Error(e); - } - } - - static { - try { - Class k = ConcurrentIntHashMap.class; - SIZECTL = Unsafe.getUnsafe().objectFieldOffset - (k.getDeclaredField("sizeCtl")); - TRANSFERINDEX = Unsafe.getUnsafe().objectFieldOffset - (k.getDeclaredField("transferIndex")); - BASECOUNT = Unsafe.getUnsafe().objectFieldOffset - (k.getDeclaredField("baseCount")); - CELLSBUSY = Unsafe.getUnsafe().objectFieldOffset - (k.getDeclaredField("cellsBusy")); - Class ck = CounterCell.class; - CELLVALUE = Unsafe.getUnsafe().objectFieldOffset - (ck.getDeclaredField("value")); - Class ak = Node[].class; - ABASE = Unsafe.getUnsafe().arrayBaseOffset(ak); - int scale = Unsafe.getUnsafe().arrayIndexScale(ak); - if ((scale & (scale - 1)) != 0) - throw new Error("data type scale not a power of two"); - ASHIFT = 31 - Integer.numberOfLeadingZeros(scale); - } catch (Exception e) { - throw new Error(e); - } - } -} diff --git a/core/src/main/java/io/questdb/client/std/FilesFacade.java b/core/src/main/java/io/questdb/client/std/FilesFacade.java deleted file mode 100644 index 97d4144..0000000 --- a/core/src/main/java/io/questdb/client/std/FilesFacade.java +++ /dev/null @@ -1,33 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.std; - -public interface FilesFacade { - boolean close(long fd); - - int errno(); - - long length(long fd); -} \ No newline at end of file diff --git a/core/src/main/java/io/questdb/client/std/GenericLexer.java b/core/src/main/java/io/questdb/client/std/GenericLexer.java deleted file mode 100644 index f5728ba..0000000 --- a/core/src/main/java/io/questdb/client/std/GenericLexer.java +++ /dev/null @@ -1,435 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.std; - -import io.questdb.client.std.str.AbstractCharSequence; -import io.questdb.client.std.str.Utf16Sink; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -import java.util.ArrayDeque; -import java.util.Comparator; - -public class GenericLexer implements ImmutableIterator, Mutable { - public static final LenComparator COMPARATOR = new LenComparator(); - public static final CharSequenceHashSet WHITESPACE = new CharSequenceHashSet(); - public static final IntHashSet WHITESPACE_CH = new IntHashSet(); - - private final ObjectPool csPairPool; - private final ObjectPool csPool; - private final ObjectPool csTriplePool; - private final CharSequence flyweightSequence = new InternalFloatingSequence(); - private final IntStack stashedNumbers = new IntStack(); - private final ArrayDeque stashedStrings = new ArrayDeque<>(); - private final IntObjHashMap> symbols = new IntObjHashMap<>(); - private final ArrayDeque unparsed = new ArrayDeque<>(); - private final IntStack unparsedPosition = new IntStack(); - private int _hi; - private int _len; - private int _lo; - private int _pos; - - private CharSequence content; - private CharSequence last; - private CharSequence next = null; - - public GenericLexer(int poolCapacity) { - csPool = new ObjectPool<>(FloatingSequence::new, poolCapacity); - csPairPool = new ObjectPool<>(FloatingSequencePair::new, poolCapacity); - csTriplePool = new ObjectPool<>(FloatingSequenceTriple::new, poolCapacity); - for (int i = 0, n = WHITESPACE.size(); i < n; i++) { - defineSymbol(Chars.toString(WHITESPACE.get(i))); - } - } - - @Override - public void clear() { - of(null, 0, 0); - - stashedNumbers.clear(); - stashedStrings.clear(); - } - - public final void defineSymbol(String token) { - char c0 = token.charAt(0); - ObjList l; - int index = symbols.keyIndex(c0); - if (index > -1) { - l = new ObjList<>(); - symbols.putAt(index, c0, l); - } else { - l = symbols.valueAtQuick(index); - } - l.add(token); - l.sort(COMPARATOR); - } - - @Override - public boolean hasNext() { - boolean n = next != null || hasUnparsed() || (content != null && _pos < _len); - if (!n && last != null) { - last = null; - } - return n; - } - - public boolean hasUnparsed() { - return !unparsed.isEmpty(); - } - - @Override - public CharSequence next() { - if (!unparsed.isEmpty()) { - this._lo = unparsedPosition.pollLast(); - this._pos = unparsedPosition.pollLast(); - - return last = unparsed.pollLast(); - } - - this._lo = this._hi; - - if (next != null) { - CharSequence result = next; - next = null; - return last = result; - } - - this._lo = this._hi = _pos; - - char term = 0; - int openTermIdx = -1; - while (_pos < _len) { - char c = content.charAt(_pos++); - CharSequence token; - switch (term) { - case 0: - switch (c) { - case '\'': - term = '\''; - openTermIdx = _pos - 1; - break; - case '"': - term = '"'; - openTermIdx = _pos - 1; - break; - case '`': - term = '`'; - openTermIdx = _pos - 1; - break; - default: - if ((token = token(c)) != null) { - return last = token; - } else { - _hi++; - } - break; - } - break; - case '\'': - if (c == '\'') { - _hi += 2; - if (_pos < _len && content.charAt(_pos) == '\'') { - _pos++; - } else { - return last = flyweightSequence; - } - } else { - _hi++; - } - break; - case '"': - if (c == '"') { - _hi += 2; - if (_pos < _len && content.charAt(_pos) == '"') { - _pos++; - } else { - return last = flyweightSequence; - } - } else { - _hi++; - } - break; - case '`': - if (c == '`') { - _hi += 2; - return last = flyweightSequence; - } else { - _hi++; - } - break; - default: - break; - } - } - if (openTermIdx != -1) { // dangling terms - if (_len == 1) { - _hi += 1; // emit term - } else { - if (openTermIdx == _lo) { // term is at the start - _hi = _lo + 1; // emit term - _pos = _hi; // rewind pos - } else if (openTermIdx == _len - 1) { // term is at the end, high is right on term - FloatingSequence termFs = csPool.next(); - termFs.lo = _hi; - termFs.hi = _hi + 1; - next = termFs; // emit term next - } else { // term is somewhere in between - _hi = openTermIdx; // emit whatever comes before term - _pos = openTermIdx; // rewind pos - } - } - } - return last = flyweightSequence; - } - - public void of(CharSequence cs, int lo, int hi) { - this.csPool.clear(); - this.csPairPool.clear(); - this.csTriplePool.clear(); - this.content = cs; - this._pos = lo; - this._len = hi; - this.next = null; - this.unparsed.clear(); - this.unparsedPosition.clear(); - this.last = null; - } - - private static CharSequence findToken0(char c, CharSequence content, int _pos, int _len, IntObjHashMap> symbols) { - final int index = symbols.keyIndex(c); - return index > -1 ? null : findToken00(content, _pos, _len, symbols, index); - } - - @Nullable - private static CharSequence findToken00(CharSequence content, int _pos, int _len, IntObjHashMap> symbols, int index) { - final ObjList l = symbols.valueAt(index); - for (int i = 0, sz = l.size(); i < sz; i++) { - CharSequence txt = l.getQuick(i); - int n = txt.length(); - boolean match = (n - 2) < (_len - _pos); - if (match) { - for (int k = 1; k < n; k++) { - if (content.charAt(_pos + (k - 1)) != txt.charAt(k)) { - match = false; - break; - } - } - } - - if (match) { - return txt; - } - } - return null; - } - - private CharSequence token(char c) { - CharSequence t = findToken0(c, content, _pos, _len, symbols); - if (t != null) { - _pos = _pos + t.length() - 1; - if (_lo == _hi) { - return t; - } - next = t; - return flyweightSequence; - } else { - return null; - } - } - - public static class FloatingSequencePair extends AbstractCharSequence implements Mutable { - public static final char NO_SEPARATOR = (char) 0; - - public FloatingSequence cs0; - public FloatingSequence cs1; - char sep = NO_SEPARATOR; - - @Override - public char charAt(int index) { - int cs0Len = cs0.length(); - if (index < cs0Len) { - return cs0.charAt(index); - } - if (sep == NO_SEPARATOR) { - return cs1.charAt(index - cs0Len); - } - return index == cs0Len ? sep : cs1.charAt(index - cs0Len - 1); - } - - @Override - public void clear() { - // no-op - } - - @Override - public int length() { - return cs0.length() + cs1.length() + (sep != NO_SEPARATOR ? 1 : 0); - } - - @NotNull - @Override - public String toString() { - final Utf16Sink b = Misc.getThreadLocalSink(); - b.put(cs0); - if (sep != NO_SEPARATOR) { - b.put(sep); - } - b.put(cs1); - return b.toString(); - } - } - - public static class FloatingSequenceTriple extends AbstractCharSequence implements Mutable { - public static final char NO_SEPARATOR = (char) 0; - - public FloatingSequence cs0; - public FloatingSequence cs1; - public FloatingSequence cs2; - char sep = NO_SEPARATOR; - - @Override - public char charAt(int index) { - int cs0Len = cs0.length(); - if (index < cs0Len) { - return cs0.charAt(index); - } - index -= cs0Len; - if (sep != NO_SEPARATOR) { - if (index == 0) { - return sep; - } - index--; - } - int cs1Len = cs1.length(); - if (index < cs1Len) { - return cs1.charAt(index); - } - index -= cs1Len; - if (sep != NO_SEPARATOR) { - if (index == 0) { - return sep; - } - index--; - } - return cs2.charAt(index); - } - - @Override - public void clear() { - // no-op - } - - @Override - public int length() { - return cs0.length() + cs1.length() + cs2.length() + (sep != NO_SEPARATOR ? 2 : 0); - } - - @NotNull - @Override - public String toString() { - final Utf16Sink b = Misc.getThreadLocalSink(); - b.put(cs0); - if (sep != NO_SEPARATOR) { - b.put(sep); - } - b.put(cs1); - if (sep != NO_SEPARATOR) { - b.put(sep); - } - b.put(cs2); - return b.toString(); - } - } - - public static class LenComparator implements Comparator { - @Override - public int compare(CharSequence o1, CharSequence o2) { - return o2.length() - o1.length(); - } - } - - public class FloatingSequence extends AbstractCharSequence implements Mutable, BufferWindowCharSequence { - int hi; - int lo; - - @Override - public char charAt(int index) { - return content.charAt(lo + index); - } - - @Override - public void clear() { - } - - @Override - public int length() { - return hi - lo; - } - - @Override - protected final CharSequence _subSequence(int start, int end) { - FloatingSequence that = csPool.next(); - that.lo = lo + start; - that.hi = lo + end; - assert that.lo <= that.hi; - return that; - } - } - - public class InternalFloatingSequence extends AbstractCharSequence { - - @Override - public char charAt(int index) { - return content.charAt(_lo + index); - } - - @Override - public int length() { - return _hi - _lo; - } - - @Override - protected CharSequence _subSequence(int start, int end) { - FloatingSequence next = csPool.next(); - next.lo = _lo + start; - next.hi = _lo + end; - assert next.lo <= next.hi; - return next; - } - - } - - static { - WHITESPACE.add(" "); - WHITESPACE.add("\t"); - WHITESPACE.add("\n"); - WHITESPACE.add("\r"); - - WHITESPACE_CH.add(' '); - WHITESPACE_CH.add('\t'); - WHITESPACE_CH.add('\n'); - WHITESPACE_CH.add('\r'); - } -} \ No newline at end of file diff --git a/core/src/main/java/io/questdb/client/std/Hash.java b/core/src/main/java/io/questdb/client/std/Hash.java index 3ded664..073a6c8 100644 --- a/core/src/main/java/io/questdb/client/std/Hash.java +++ b/core/src/main/java/io/questdb/client/std/Hash.java @@ -26,19 +26,8 @@ public final class Hash { - // Constant from Rust compiler's FxHasher. - private static final long M2 = 0x517cc1b727220a95L; - private static final int SPREAD_HASH_BITS = 0x7fffffff; - public static int hashLong128_32(long key1, long key2) { - return (int) hashLong128_64(key1, key2); - } - - public static long hashLong128_64(long key1, long key2) { - return fmix64(key1 * M2 + key2); - } - public static int hashLong32(long k) { return (int) hashLong64(k); } diff --git a/core/src/main/java/io/questdb/client/std/ImmutableIterator.java b/core/src/main/java/io/questdb/client/std/ImmutableIterator.java deleted file mode 100644 index 44fbda0..0000000 --- a/core/src/main/java/io/questdb/client/std/ImmutableIterator.java +++ /dev/null @@ -1,38 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.std; - -import org.jetbrains.annotations.NotNull; - -import java.util.Iterator; - -public interface ImmutableIterator extends Iterator, Iterable { - - @Override - @NotNull - default Iterator iterator() { - return this; - } -} \ No newline at end of file diff --git a/core/src/main/java/io/questdb/client/std/CharSequenceHashSet.java b/core/src/main/java/io/questdb/client/std/LongHashSet.java similarity index 56% rename from core/src/main/java/io/questdb/client/std/CharSequenceHashSet.java rename to core/src/main/java/io/questdb/client/std/LongHashSet.java index 4cb1f32..66335c2 100644 --- a/core/src/main/java/io/questdb/client/std/CharSequenceHashSet.java +++ b/core/src/main/java/io/questdb/client/std/LongHashSet.java @@ -27,40 +27,37 @@ import io.questdb.client.std.str.CharSink; import io.questdb.client.std.str.Sinkable; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import java.util.Arrays; -public class CharSequenceHashSet extends AbstractCharSequenceHashSet implements Sinkable { + +public class LongHashSet extends AbstractLongHashSet implements Sinkable { + + public static final double DEFAULT_LOAD_FACTOR = 0.4; private static final int MIN_INITIAL_CAPACITY = 16; - private final ObjList list; - private boolean hasNull = false; + private final LongList list; - public CharSequenceHashSet() { + public LongHashSet() { this(MIN_INITIAL_CAPACITY); } - private CharSequenceHashSet(int initialCapacity) { - this(initialCapacity, 0.4); + public LongHashSet(int initialCapacity) { + this(initialCapacity, DEFAULT_LOAD_FACTOR, noEntryKey); } - public CharSequenceHashSet(int initialCapacity, double loadFactor) { - super(initialCapacity, loadFactor); - list = new ObjList<>(free); + public LongHashSet(int initialCapacity, double loadFactor, long noKeyValue) { + super(initialCapacity, loadFactor, noKeyValue); + list = new LongList(free); clear(); } /** * Adds key to hash set preserving key uniqueness. * - * @param key immutable sequence of characters. + * @param key key to be added. * @return false if key is already in the set and true otherwise. */ - public boolean add(@Nullable CharSequence key) { - if (key == null) { - return addNull(); - } - + public boolean add(long key) { int index = keyIndex(key); if (index < 0) { return false; @@ -70,45 +67,43 @@ public boolean add(@Nullable CharSequence key) { return true; } - public void addAt(int index, @NotNull CharSequence key) { - final String s = Chars.toString(key); - keys[index] = s; - list.add(s); + public void addAt(int index, long key) { + keys[index] = key; + list.add(key); if (--free < 1) { rehash(); } } - public boolean addNull() { - if (hasNull) { - return false; - } - --free; - hasNull = true; - list.add(null); - return true; - } - - @Override public final void clear() { free = capacity; - Arrays.fill(keys, null); + Arrays.fill(keys, noEntryKeyValue); list.clear(); - hasNull = false; } - @Override - public boolean contains(@Nullable CharSequence key) { - return key == null ? hasNull : keyIndex(key) < 0; + public boolean contains(long key) { + return keyIndex(key) < 0; } - public CharSequence get(int index) { + public long get(int index) { return list.getQuick(index); } + public long getLast() { + return list.getLast(); + } + + public void removeAt(int index) { + if (index < 0) { + long key = keys[-index - 1]; + super.removeAt(index); + listRemove(key); + } + } + @Override public void toSink(@NotNull CharSink sink) { - sink.put(list); + list.toSink(sink); } @Override @@ -116,18 +111,45 @@ public String toString() { return list.toString(); } + private void listRemove(long v) { + int sz = list.size(); + for (int i = 0; i < sz; i++) { + if (list.getQuick(i) == v) { + // shift remaining elements left + for (int j = i + 1; j < sz; j++) { + list.setQuick(j - 1, list.getQuick(j)); + } + list.setPos(sz - 1); + return; + } + } + } + private void rehash() { int newCapacity = capacity * 2; free = capacity = newCapacity; int len = Numbers.ceilPow2((int) (newCapacity / loadFactor)); - this.keys = new CharSequence[len]; + this.keys = new long[len]; + Arrays.fill(keys, noEntryKeyValue); mask = len - 1; int n = list.size(); free -= n; for (int i = 0; i < n; i++) { - final CharSequence key = list.getQuick(i); - keys[keyIndex(key)] = key; + long key = list.getQuick(i); + int keyIndex = keyIndex(key); + keys[keyIndex] = key; } } -} \ No newline at end of file + @Override + protected void erase(int index) { + keys[index] = noEntryKeyValue; + } + + @Override + protected void move(int from, int to) { + keys[to] = keys[from]; + erase(from); + } + +} diff --git a/core/src/main/java/io/questdb/client/std/LongObjHashMap.java b/core/src/main/java/io/questdb/client/std/LongObjHashMap.java deleted file mode 100644 index b927efe..0000000 --- a/core/src/main/java/io/questdb/client/std/LongObjHashMap.java +++ /dev/null @@ -1,110 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.std; - -import java.util.Arrays; - -public class LongObjHashMap extends AbstractLongHashSet { - private V[] values; - - public LongObjHashMap() { - this(8); - } - - public LongObjHashMap(int initialCapacity) { - this(initialCapacity, 0.5f); - } - - @SuppressWarnings("unchecked") - private LongObjHashMap(int initialCapacity, double loadFactor) { - super(initialCapacity, loadFactor); - values = (V[]) new Object[keys.length]; - clear(); - } - - @Override - public void clear() { - super.clear(); - Arrays.fill(values, null); - } - - public void putAt(int index, long key, V value) { - if (index < 0) { - values[-index - 1] = value; - } else { - keys[index] = key; - values[index] = value; - if (--free == 0) { - rehash(); - } - } - } - - public V valueAt(int index) { - return index < 0 ? valueAtQuick(index) : null; - } - - public V valueAtQuick(int index) { - return values[-index - 1]; - } - - @SuppressWarnings("unchecked") - private void rehash() { - int size = size(); - int newCapacity = capacity * 2; - free = capacity = newCapacity; - int len = Numbers.ceilPow2((int) (newCapacity / loadFactor)); - - V[] oldValues = values; - long[] oldKeys = keys; - this.keys = new long[len]; - this.values = (V[]) new Object[len]; - Arrays.fill(keys, noEntryKeyValue); - mask = len - 1; - - free -= size; - for (int i = oldKeys.length; i-- > 0; ) { - long key = oldKeys[i]; - if (key != noEntryKeyValue) { - final int index = keyIndex(key); - keys[index] = key; - values[index] = oldValues[i]; - } - } - } - - @Override - protected void erase(int index) { - keys[index] = this.noEntryKeyValue; - } - - @Override - protected void move(int from, int to) { - keys[to] = keys[from]; - values[to] = values[from]; - erase(from); - } - -} \ No newline at end of file diff --git a/core/src/main/java/io/questdb/client/std/LowerCaseAsciiCharSequenceIntHashMap.java b/core/src/main/java/io/questdb/client/std/LowerCaseAsciiCharSequenceIntHashMap.java index e833b38..ab70edb 100644 --- a/core/src/main/java/io/questdb/client/std/LowerCaseAsciiCharSequenceIntHashMap.java +++ b/core/src/main/java/io/questdb/client/std/LowerCaseAsciiCharSequenceIntHashMap.java @@ -65,14 +65,14 @@ public boolean putAt(int index, CharSequence key, int value) { values[-index - 1] = value; return false; } - putAt0(index, Chars.toLowerCaseAscii(key), value); + putAt0(index, Chars.toLowerCase(key), value); return true; } public void putIfAbsent(CharSequence key, int value) { int index = keyIndex(key); if (index > -1) { - putAt0(index, Chars.toLowerCaseAscii(key), value); + putAt0(index, Chars.toLowerCase(key), value); } } diff --git a/core/src/main/java/io/questdb/client/std/LowerCaseCharSequenceHashSet.java b/core/src/main/java/io/questdb/client/std/LowerCaseCharSequenceHashSet.java deleted file mode 100644 index 3795466..0000000 --- a/core/src/main/java/io/questdb/client/std/LowerCaseCharSequenceHashSet.java +++ /dev/null @@ -1,106 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.std; - -public class LowerCaseCharSequenceHashSet extends AbstractLowerCaseCharSequenceHashSet { - private static final int MIN_INITIAL_CAPACITY = 16; - - public LowerCaseCharSequenceHashSet() { - this(MIN_INITIAL_CAPACITY); - } - - private LowerCaseCharSequenceHashSet(int initialCapacity) { - this(initialCapacity, 0.4); - } - - private LowerCaseCharSequenceHashSet(int initialCapacity, double loadFactor) { - super(initialCapacity, loadFactor); - clear(); - } - - /** - * Adds key to hash set preserving key uniqueness. - * - * @param key immutable sequence of characters. - * @return false if key is already in the set and true otherwise. - */ - public boolean add(CharSequence key) { - int index = keyIndex(key); - if (index < 0) { - return false; - } - - addAt(index, key); - return true; - } - - public void addAt(int index, CharSequence key) { - keys[index] = key; - if (--free < 1) { - rehash(); - } - } - - // returns the first non-null key, in arbitrary order - public CharSequence getAny() { - for (int i = 0, n = keys.length; i < n; i++) { - if (keys[i] != noEntryKey) { - return keys[i]; - } - } - return null; - } - - public CharSequence keyAt(int index) { - return keys[-index - 1]; - } - - private void rehash() { - int newCapacity = capacity * 2; - final int size = size(); - free = capacity = newCapacity; - int len = Numbers.ceilPow2((int) (newCapacity / loadFactor)); - CharSequence[] newKeys = new CharSequence[len]; - CharSequence[] oldKeys = keys; - mask = len - 1; - this.keys = newKeys; - free -= size; - for (int i = 0, n = oldKeys.length; i < n; i++) { - CharSequence key = oldKeys[i]; - if (key != null) { - keys[keyIndex(key)] = key; - } - } - } - - protected void erase(int index) { - keys[index] = noEntryKey; - } - - protected void move(int from, int to) { - keys[to] = keys[from]; - erase(from); - } -} diff --git a/core/src/main/java/io/questdb/client/std/Misc.java b/core/src/main/java/io/questdb/client/std/Misc.java index 87797c4..e57d814 100644 --- a/core/src/main/java/io/questdb/client/std/Misc.java +++ b/core/src/main/java/io/questdb/client/std/Misc.java @@ -30,7 +30,6 @@ import java.io.Closeable; import java.io.IOException; -import java.util.Arrays; public final class Misc { public static final String EOL = "\r\n"; @@ -100,12 +99,6 @@ public static Utf8StringSink getThreadLocalUtf8Sink() { return b; } - public static int[] getWorkerAffinity(int workerCount) { - int[] res = new int[workerCount]; - Arrays.fill(res, -1); - return res; - } - private static void freeObjList0(ObjList list) { for (int i = 0, n = list.size(); i < n; i++) { list.setQuick(i, freeIfCloseable(list.getQuick(i))); diff --git a/core/src/main/java/io/questdb/client/std/Numbers.java b/core/src/main/java/io/questdb/client/std/Numbers.java index 585df73..e35e81d 100644 --- a/core/src/main/java/io/questdb/client/std/Numbers.java +++ b/core/src/main/java/io/questdb/client/std/Numbers.java @@ -25,7 +25,6 @@ package io.questdb.client.std; import io.questdb.client.std.fastdouble.FastDoubleParser; -import io.questdb.client.std.fastdouble.FastFloatParser; import io.questdb.client.std.str.CharSink; import io.questdb.client.std.str.Utf8Sequence; import jdk.internal.math.FDBigInteger; @@ -263,43 +262,6 @@ public static void appendHex(CharSink sink, long value, boolean pad) { array[bit].append(sink, value); } - /** - * Append a long value to a CharSink in hex format. - * - * @param sink the CharSink to append to - * @param value the value to append - * @param padToBytes if non-zero, pad the output to the specified number of bytes - */ - public static void appendHexPadded(CharSink sink, long value, int padToBytes) { - assert padToBytes >= 0 && padToBytes <= 8; - // This code might be unclear, so here are some hints: - // This method uses longHexAppender() and longHexAppender() is always padding to a whole byte. It never prints - // just a nibble. It means the longHexAppender() will print value 0xf as "0f". Value 0xff will be printed as "ff". - // Value 0xfff will be printed as "0fff". Value 0xffff will be printed as "ffff" and so on. - // So this method needs to pad only from the next whole byte up. - // In other words: This method always pads with full bytes (=even number of zeros), never with just a nibble. - - // Example 1: Value is 0xF and padToBytes is 2. This means the desired output is 000f. - // longHexAppender() pads to a full byte. This means it will output is 0f. So this method needs to pad with 2 zeros. - - // Example 2: The value is 0xFF and padToBytes is 2. This means the desired output is 00ff. - // longHexAppender() will output "ff". This is a full byte so longHexAppender() will not do any padding on its own. - // So this method needs to pad with 2 zeros. - int leadingZeroBits = Long.numberOfLeadingZeros(value); - int padToBits = padToBytes << 3; - int bitsToPad = padToBits - (Long.SIZE - leadingZeroBits); - int bytesToPad = (bitsToPad >> 3); - for (int i = 0; i < bytesToPad; i++) { - sink.putAscii('0'); - sink.putAscii('0'); - } - if (value == 0) { - return; - } - int bit = 64 - leadingZeroBits; - longHexAppender[bit].append(sink, value); - } - public static void appendHexPadded(CharSink sink, final int value) { int i = value; if (i < 0) { @@ -386,38 +348,6 @@ public static void appendHexPadded(CharSink sink, final int value) { } } - public static void appendLong256(long a, long b, long c, long d, CharSink sink) { - if (a == Numbers.LONG_NULL && b == Numbers.LONG_NULL && c == Numbers.LONG_NULL && d == Numbers.LONG_NULL) { - return; - } - sink.putAscii("0x"); - if (d != 0) { - appendLong256Four(a, b, c, d, sink); - return; - } - if (c != 0) { - appendLong256Three(a, b, c, sink); - return; - } - if (b != 0) { - appendLong256Two(a, b, sink); - return; - } - appendHex(sink, a, false); - } - - public static void appendUuid(long lo, long hi, CharSink sink) { - appendHexPadded(sink, (hi >> 32) & 0xFFFFFFFFL, 4); - sink.putAscii('-'); - appendHexPadded(sink, (hi >> 16) & 0xFFFF, 2); - sink.putAscii('-'); - appendHexPadded(sink, hi & 0xFFFF, 2); - sink.putAscii('-'); - appendHexPadded(sink, lo >> 48 & 0xFFFF, 2); - sink.putAscii('-'); - appendHexPadded(sink, lo & 0xFFFFFFFFFFFFL, 6); - } - public static int ceilPow2(int value) { int i = value; if ((i != 0) && (i & (i - 1)) > 0) { @@ -459,17 +389,6 @@ public static int hexToDecimal(int c) throws NumericException { return r; } - public static void intToIPv4Sink(CharSink sink, int value) { - // NULL handling should be done outside, null here will be printed as 0.0.0.0 - append(sink, (value >> 24) & 0xff); - sink.putAscii('.'); - append(sink, (value >> 16) & 0xff); - sink.putAscii('.'); - append(sink, (value >> 8) & 0xff); - sink.putAscii('.'); - append(sink, value & 0xff); - } - public static long interleaveBits(long x, long y) { return spreadBits(x) | (spreadBits(y) << 1); } @@ -490,10 +409,6 @@ public static boolean isNull(float value) { return Float.isNaN(value) || Float.isInfinite(value); } - public static boolean isPow2(int value) { - return value > 0 && (value & (value - 1)) == 0; - } - public static int msb(int value) { return 31 - Integer.numberOfLeadingZeros(value); } @@ -506,10 +421,6 @@ public static double parseDouble(CharSequence sequence) throws NumericException return FastDoubleParser.parseDouble(sequence, true); } - public static float parseFloat(CharSequence sequence) throws NumericException { - return FastFloatParser.parseFloat(sequence, true); - } - public static int parseHexInt(CharSequence sequence) throws NumericException { return parseHexInt(sequence, 0, sequence.length()); } @@ -557,17 +468,6 @@ public static int parseIPv4(CharSequence sequence) throws NumericException { return parseIPv4_0(sequence, 0, sequence.length()); } - public static int parseIPv4Quiet(CharSequence sequence) { - try { - if (sequence == null || Chars.equals("null", sequence)) { - return IPv4_NULL; - } - return parseIPv4(sequence); - } catch (NumericException e) { - return IPv4_NULL; - } - } - public static int parseIPv4_0(CharSequence sequence, final int p, int lim) throws NumericException { if (lim == 0) { throw NumericException.instance().put("empty IPv4 address string"); @@ -657,101 +557,6 @@ public static int parseInt(CharSequence sequence, int p, int lim) throws Numeric return parseInt0(sequence, p, lim); } - public static long parseInt000Greedy(CharSequence sequence, final int p, int lim) throws NumericException { - if (lim == p) { - throw NumericException.instance().put("empty number string"); - } - - boolean negative = sequence.charAt(p) == '-'; - int i = p; - if (negative) { - i++; - } - - if (i >= lim || notDigit(sequence.charAt(i))) { - throw NumericException.instance().put("not a number: ").put(sequence); - } - - int val = 0; - for (; i < lim; i++) { - char c = sequence.charAt(i); - - if (notDigit(c)) { - break; - } - - // val * 10 + (c - '0') - int r = (val << 3) + (val << 1) - (c - '0'); - if (r > val) { - throw NumericException.instance().put("number overflow"); - } - val = r; - } - - final int len = i - p; - - if (len > 3 || val == Integer.MIN_VALUE && !negative) { - throw NumericException.instance().put("number overflow"); - } - - while (i - p < 3) { - val *= 10; - i++; - } - - return encodeLowHighInts(negative ? val : -val, len); - } - - public static int parseIntQuiet(CharSequence sequence) { - try { - if (sequence == null || Chars.equals("NaN", sequence)) { - return Numbers.INT_NULL; - } - return parseInt0(sequence, 0, sequence.length()); - } catch (NumericException e) { - return Numbers.INT_NULL; - } - - } - - public static long parseIntSafely(CharSequence sequence, final int p, int lim) throws NumericException { - if (lim == p) { - throw NumericException.instance().put("empty number string"); - } - - boolean negative = sequence.charAt(p) == '-'; - int i = p; - if (negative) { - i++; - } - - if (i >= lim || notDigit(sequence.charAt(i))) { - throw NumericException.instance().put("not a number: ").put(sequence); - } - - int val = 0; - for (; i < lim; i++) { - char c = sequence.charAt(i); - - if (notDigit(c)) { - break; - } - - // val * 10 + (c - '0') - int r = (val << 3) + (val << 1) - (c - '0'); - if (r > val) { - throw NumericException.instance().put("number overflow"); - } - val = r; - } - - if (val == Integer.MIN_VALUE && !negative) { - throw NumericException.instance().put("number overflow"); - } - - return encodeLowHighInts(negative ? val : -val, i - p); - } - public static long parseLong(CharSequence sequence) throws NumericException { if (sequence == null) { throw NumericException.instance().put("null string"); @@ -766,51 +571,6 @@ public static long parseLong(Utf8Sequence sequence) throws NumericException { return parseLong0(sequence.asAsciiCharSequence(), 0, sequence.size()); } - public static long parseLong000000Greedy(CharSequence sequence, final int p, int lim) throws NumericException { - if (lim == p) { - throw NumericException.instance().put("empty number string"); - } - - boolean negative = sequence.charAt(p) == '-'; - int i = p; - if (negative) { - i++; - } - - if (i >= lim || notDigit(sequence.charAt(i))) { - throw NumericException.instance().put("not a number: ").put(sequence); - } - - int val = 0; - for (; i < lim; i++) { - char c = sequence.charAt(i); - - if (notDigit(c)) { - break; - } - - // val * 10 + (c - '0') - int r = (val << 3) + (val << 1) - (c - '0'); - if (r > val) { - throw NumericException.instance().put("number overflow"); - } - val = r; - } - - final int len = i - p; - - if (len > 6 || val == Integer.MIN_VALUE && !negative) { - throw NumericException.instance().put("number overflow"); - } - - while (i - p < 6) { - val *= 10; - i++; - } - - return encodeLowHighInts(negative ? val : -val, len); - } - public static long spreadBits(long v) { v = (v | (v << 16)) & 0X0000FFFF0000FFFFL; v = (v | (v << 8)) & 0X00FF00FF00FF00FFL; @@ -1440,21 +1200,6 @@ private static void appendLong2(CharSink sink, long i) { sink.putAscii((char) ('0' + i % 10)); } - private static void appendLong256Four(long a, long b, long c, long d, CharSink sink) { - appendLong256Three(b, c, d, sink); - appendHex(sink, a, true); - } - - private static void appendLong256Three(long a, long b, long c, CharSink sink) { - appendLong256Two(b, c, sink); - appendHex(sink, a, true); - } - - private static void appendLong256Two(long a, long b, CharSink sink) { - appendHex(sink, b, false); - appendHex(sink, a, true); - } - private static void appendLong3(CharSink sink, long i) { long c; sink.putAscii((char) ('0' + i / 100)); diff --git a/core/src/main/java/io/questdb/client/std/Rnd.java b/core/src/main/java/io/questdb/client/std/Rnd.java index 8c9f4f5..2237a38 100644 --- a/core/src/main/java/io/questdb/client/std/Rnd.java +++ b/core/src/main/java/io/questdb/client/std/Rnd.java @@ -24,7 +24,6 @@ package io.questdb.client.std; -import io.questdb.client.cairo.GeoHashes; import io.questdb.client.std.str.StringSink; import io.questdb.client.std.str.Utf16Sink; @@ -187,17 +186,6 @@ public double nextDouble() { return (((long) (nextIntForDouble(26)) << 27) + nextIntForDouble(27)) * DOUBLE_UNIT; } - public long nextGeoHash(int bits) { - double x = nextDouble() * 180.0 - 90.0; - double y = nextDouble() * 360.0 - 180.0; - try { - return GeoHashes.fromCoordinatesDeg(x, y, bits); - } catch (NumericException e) { - // Should never happen - return GeoHashes.NULL; - } - } - public int nextInt(int boundary) { return nextPositiveInt() % boundary; } diff --git a/core/src/main/java/io/questdb/client/std/SecureRnd.java b/core/src/main/java/io/questdb/client/std/SecureRnd.java new file mode 100644 index 0000000..91c2609 --- /dev/null +++ b/core/src/main/java/io/questdb/client/std/SecureRnd.java @@ -0,0 +1,185 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.std; + +import java.security.SecureRandom; + +/** + * Zero-GC cryptographically secure random number generator based on ChaCha20 + * in counter mode (RFC 7539). Seeded once from {@link SecureRandom} at + * construction time, then produces unpredictable output with no heap + * allocations. + *

    + * Each {@link #nextInt()} call returns one 32-bit word from the ChaCha20 + * keystream. A single block computation yields 16 words, so the amortized + * cost is one ChaCha20 block per 16 calls. + */ +public class SecureRnd { + + // "expand 32-byte k" in little-endian + private static final int CONSTANT_0 = 0x61707865; + private static final int CONSTANT_1 = 0x3320646e; + private static final int CONSTANT_2 = 0x79622d32; + private static final int CONSTANT_3 = 0x6b206574; + + private final int[] output = new int[16]; + private final int[] state = new int[16]; + private int outputPos = 16; // forces block computation on first call + + /** + * Creates a new instance seeded from {@link SecureRandom}. + */ + public SecureRnd() { + SecureRandom seed = new SecureRandom(); + byte[] seedBytes = new byte[44]; // 32 (key) + 12 (nonce) + seed.nextBytes(seedBytes); + init(seedBytes, 0); + } + + /** + * Creates a new instance with an explicit key, nonce, and initial counter. + * Useful for testing with known RFC 7539 test vectors. + * + * @param key 32-byte key + * @param nonce 12-byte nonce + * @param counter initial block counter value + */ + public SecureRnd(byte[] key, byte[] nonce, int counter) { + byte[] seedBytes = new byte[44]; + System.arraycopy(key, 0, seedBytes, 0, 32); + System.arraycopy(nonce, 0, seedBytes, 32, 12); + init(seedBytes, counter); + } + + /** + * Returns the next cryptographically secure random int. + */ + public int nextInt() { + if (outputPos >= 16) { + computeBlock(); + outputPos = 0; + } + return output[outputPos++]; + } + + private void computeBlock() { + int x0 = state[0], x1 = state[1], x2 = state[2], x3 = state[3]; + int x4 = state[4], x5 = state[5], x6 = state[6], x7 = state[7]; + int x8 = state[8], x9 = state[9], x10 = state[10], x11 = state[11]; + int x12 = state[12], x13 = state[13], x14 = state[14], x15 = state[15]; + + for (int i = 0; i < 10; i++) { + // Column rounds + x0 += x4; x12 ^= x0; x12 = Integer.rotateLeft(x12, 16); + x8 += x12; x4 ^= x8; x4 = Integer.rotateLeft(x4, 12); + x0 += x4; x12 ^= x0; x12 = Integer.rotateLeft(x12, 8); + x8 += x12; x4 ^= x8; x4 = Integer.rotateLeft(x4, 7); + + x1 += x5; x13 ^= x1; x13 = Integer.rotateLeft(x13, 16); + x9 += x13; x5 ^= x9; x5 = Integer.rotateLeft(x5, 12); + x1 += x5; x13 ^= x1; x13 = Integer.rotateLeft(x13, 8); + x9 += x13; x5 ^= x9; x5 = Integer.rotateLeft(x5, 7); + + x2 += x6; x14 ^= x2; x14 = Integer.rotateLeft(x14, 16); + x10 += x14; x6 ^= x10; x6 = Integer.rotateLeft(x6, 12); + x2 += x6; x14 ^= x2; x14 = Integer.rotateLeft(x14, 8); + x10 += x14; x6 ^= x10; x6 = Integer.rotateLeft(x6, 7); + + x3 += x7; x15 ^= x3; x15 = Integer.rotateLeft(x15, 16); + x11 += x15; x7 ^= x11; x7 = Integer.rotateLeft(x7, 12); + x3 += x7; x15 ^= x3; x15 = Integer.rotateLeft(x15, 8); + x11 += x15; x7 ^= x11; x7 = Integer.rotateLeft(x7, 7); + + // Diagonal rounds + x0 += x5; x15 ^= x0; x15 = Integer.rotateLeft(x15, 16); + x10 += x15; x5 ^= x10; x5 = Integer.rotateLeft(x5, 12); + x0 += x5; x15 ^= x0; x15 = Integer.rotateLeft(x15, 8); + x10 += x15; x5 ^= x10; x5 = Integer.rotateLeft(x5, 7); + + x1 += x6; x12 ^= x1; x12 = Integer.rotateLeft(x12, 16); + x11 += x12; x6 ^= x11; x6 = Integer.rotateLeft(x6, 12); + x1 += x6; x12 ^= x1; x12 = Integer.rotateLeft(x12, 8); + x11 += x12; x6 ^= x11; x6 = Integer.rotateLeft(x6, 7); + + x2 += x7; x13 ^= x2; x13 = Integer.rotateLeft(x13, 16); + x8 += x13; x7 ^= x8; x7 = Integer.rotateLeft(x7, 12); + x2 += x7; x13 ^= x2; x13 = Integer.rotateLeft(x13, 8); + x8 += x13; x7 ^= x8; x7 = Integer.rotateLeft(x7, 7); + + x3 += x4; x14 ^= x3; x14 = Integer.rotateLeft(x14, 16); + x9 += x14; x4 ^= x9; x4 = Integer.rotateLeft(x4, 12); + x3 += x4; x14 ^= x3; x14 = Integer.rotateLeft(x14, 8); + x9 += x14; x4 ^= x9; x4 = Integer.rotateLeft(x4, 7); + } + + // Feed-forward: add original state + output[0] = x0 + state[0]; + output[1] = x1 + state[1]; + output[2] = x2 + state[2]; + output[3] = x3 + state[3]; + output[4] = x4 + state[4]; + output[5] = x5 + state[5]; + output[6] = x6 + state[6]; + output[7] = x7 + state[7]; + output[8] = x8 + state[8]; + output[9] = x9 + state[9]; + output[10] = x10 + state[10]; + output[11] = x11 + state[11]; + output[12] = x12 + state[12]; + output[13] = x13 + state[13]; + output[14] = x14 + state[14]; + output[15] = x15 + state[15]; + + // Increment block counter + state[12]++; + } + + private void init(byte[] seedBytes, int counter) { + state[0] = CONSTANT_0; + state[1] = CONSTANT_1; + state[2] = CONSTANT_2; + state[3] = CONSTANT_3; + + // Key: 8 little-endian ints from seedBytes[0..31] + for (int i = 0; i < 8; i++) { + int off = i * 4; + state[4 + i] = (seedBytes[off] & 0xFF) + | ((seedBytes[off + 1] & 0xFF) << 8) + | ((seedBytes[off + 2] & 0xFF) << 16) + | ((seedBytes[off + 3] & 0xFF) << 24); + } + + state[12] = counter; + + // Nonce: 3 little-endian ints from seedBytes[32..43] + for (int i = 0; i < 3; i++) { + int off = 32 + i * 4; + state[13 + i] = (seedBytes[off] & 0xFF) + | ((seedBytes[off + 1] & 0xFF) << 8) + | ((seedBytes[off + 2] & 0xFF) << 16) + | ((seedBytes[off + 3] & 0xFF) << 24); + } + } +} diff --git a/core/src/main/java/io/questdb/client/std/Unsafe.java b/core/src/main/java/io/questdb/client/std/Unsafe.java index d8e8850..b5d483e 100644 --- a/core/src/main/java/io/questdb/client/std/Unsafe.java +++ b/core/src/main/java/io/questdb/client/std/Unsafe.java @@ -24,42 +24,31 @@ package io.questdb.client.std; -// @formatter:off import io.questdb.client.cairo.CairoException; -import org.jetbrains.annotations.Nullable; -import java.lang.invoke.MethodHandles; import java.lang.reflect.AccessibleObject; -import java.lang.reflect.Array; import java.lang.reflect.Field; import java.lang.reflect.Method; import java.util.concurrent.atomic.LongAdder; -import static io.questdb.client.std.MemoryTag.NATIVE_DEFAULT; - public final class Unsafe { // The various _ADDR fields are `long` in Java, but they are `* mut usize` in Rust, or `size_t*` in C. // These are off-heap allocated atomic counters for memory usage tracking. public static final long BYTE_OFFSET; public static final long BYTE_SCALE; - public static final long INT_OFFSET; - public static final long INT_SCALE; public static final Module JAVA_BASE_MODULE = System.class.getModule(); public static final long LONG_OFFSET; public static final long LONG_SCALE; private static final LongAdder[] COUNTERS = new LongAdder[MemoryTag.SIZE]; private static final long FREE_COUNT_ADDR; private static final long MALLOC_COUNT_ADDR; - private static final long[] NATIVE_ALLOCATORS = new long[MemoryTag.SIZE - NATIVE_DEFAULT]; private static final long[] NATIVE_MEM_COUNTER_ADDRS = new long[MemoryTag.SIZE]; private static final long NON_RSS_MEM_USED_ADDR; private static final long OVERRIDE; private static final long REALLOC_COUNT_ADDR; - private static final long RSS_MEM_LIMIT_ADDR; private static final long RSS_MEM_USED_ADDR; private static final sun.misc.Unsafe UNSAFE; - private static final AnonymousClassDefiner anonymousClassDefiner; private static final Method implAddExports; private Unsafe() { @@ -73,40 +62,6 @@ public static void addExports(Module from, Module to, String packageName) { } } - public static long arrayGetVolatile(long[] array, int index) { - assert index > -1 && index < array.length; - return Unsafe.getUnsafe().getLongVolatile(array, LONG_OFFSET + ((long) index << LONG_SCALE)); - } - - public static int arrayGetVolatile(int[] array, int index) { - assert index > -1 && index < array.length; - return Unsafe.getUnsafe().getIntVolatile(array, INT_OFFSET + ((long) index << INT_SCALE)); - } - - /** - * This call has Atomic*#lazySet / memory_order_release semantics. - * - * @param array array to put into - * @param index index - * @param value value to put - */ - public static void arrayPutOrdered(long[] array, int index, long value) { - assert index > -1 && index < array.length; - Unsafe.getUnsafe().putOrderedLong(array, LONG_OFFSET + ((long) index << LONG_SCALE), value); - } - - /** - * This call has Atomic*#lazySet / memory_order_release semantics. - * - * @param array array to put into - * @param index index - * @param value value to put - */ - public static void arrayPutOrdered(int[] array, int index, int value) { - assert index > -1 && index < array.length; - Unsafe.getUnsafe().putOrderedInt(array, INT_OFFSET + ((long) index << INT_SCALE), value); - } - public static int byteArrayGetInt(byte[] array, int index) { assert index > -1 && index < array.length - 3; return Unsafe.getUnsafe().getInt(array, BYTE_OFFSET + index); @@ -141,21 +96,6 @@ public static boolean cas(long[] array, int index, long expected, long value) { return Unsafe.cas(array, Unsafe.LONG_OFFSET + (((long) index) << Unsafe.LONG_SCALE), expected, value); } - /** - * Defines a class but does not make it known to the class loader or system dictionary. - *

    - * Equivalent to {@code Unsafe#defineAnonymousClass} and {@code Lookup#defineHiddenClass}, except that - * it does not support constant pool patches. - * - * @param hostClass context for linkage, access control, protection domain, and class loader - * @param data bytes of a class file - * @return Java Class for the given bytecode - */ - @Nullable - public static Class defineAnonymousClass(Class hostClass, byte[] data) { - return anonymousClassDefiner.define(hostClass, data); - } - public static long free(long ptr, long size, int memoryTag) { if (ptr != 0) { Unsafe.getUnsafe().freeMemory(ptr); @@ -199,19 +139,10 @@ public static long getMemUsedByTag(int memoryTag) { return COUNTERS[memoryTag].sum() + UNSAFE.getLongVolatile(null, NATIVE_MEM_COUNTER_ADDRS[memoryTag]); } - /** Returns a `*const QdbAllocator` for use in Rust. */ - public static long getNativeAllocator(int memoryTag) { - return NATIVE_ALLOCATORS[memoryTag - NATIVE_DEFAULT]; - } - public static long getReallocCount() { return UNSAFE.getLongVolatile(null, REALLOC_COUNT_ADDR); } - public static long getRssMemLimit() { - return UNSAFE.getLongVolatile(null, RSS_MEM_LIMIT_ADDR); - } - public static long getRssMemUsed() { return UNSAFE.getLongVolatile(null, RSS_MEM_USED_ADDR); } @@ -245,7 +176,6 @@ public static void makeAccessible(AccessibleObject accessibleObject) { public static long malloc(long size, int memoryTag) { try { assert memoryTag >= MemoryTag.NATIVE_PATH; - checkAllocLimit(size, memoryTag); long ptr = Unsafe.getUnsafe().allocateMemory(size); recordMemAlloc(size, memoryTag); incrMallocCount(); @@ -267,7 +197,6 @@ public static long malloc(long size, int memoryTag) { public static long realloc(long address, long oldSize, long newSize, int memoryTag) { try { assert memoryTag >= MemoryTag.NATIVE_PATH; - checkAllocLimit(-oldSize + newSize, memoryTag); long ptr = Unsafe.getUnsafe().reallocateMemory(address, newSize); recordMemAlloc(-oldSize + newSize, memoryTag); incrReallocCount(); @@ -300,10 +229,6 @@ public static void recordMemAlloc(long size, int memoryTag) { } } - public static void setRssMemLimit(long limit) { - UNSAFE.putLongVolatile(null, RSS_MEM_LIMIT_ADDR, limit); - } - private static long AccessibleObject_override_fieldOffset() { if (isJava8Or11()) { return getFieldOffset(AccessibleObject.class, "override"); @@ -319,39 +244,6 @@ private static long AccessibleObject_override_fieldOffset() { return 16L; } - private static void checkAllocLimit(long size, int memoryTag) { - if (size <= 0) { - return; - } - // Don't check limits for mmap'd memory - final long rssMemLimit = getRssMemLimit(); - if (rssMemLimit > 0 && memoryTag >= NATIVE_DEFAULT) { - long usage = getRssMemUsed(); - if (usage + size > rssMemLimit) { - throw CairoException.nonCritical() - .put("global RSS memory limit exceeded [usage=") - .put(usage) - .put(", RSS_MEM_LIMIT=").put(rssMemLimit) - .put(", size=").put(size) - .put(", memoryTag=").put(memoryTag) - .put(']'); - } - } - } - - /** Allocate a new native allocator object and return its pointer */ - private static long constructNativeAllocator(long nativeMemCountersArray, int memoryTag) { - // See `allocator.rs` for the definition of `QdbAllocator`. - // We construct here via `Unsafe` to avoid having initialization order issues with `Os.java`. - final long allocSize = 8 + 8 + 4; // two longs, one int - final long addr = UNSAFE.allocateMemory(allocSize); - Vect.memset(addr, allocSize, 0); - UNSAFE.putLong(addr, nativeMemCountersArray); - UNSAFE.putLong(addr + 8, NATIVE_MEM_COUNTER_ADDRS[memoryTag]); - UNSAFE.putInt(addr + 16, memoryTag); - return addr; - } - private static boolean getOrdinaryObjectPointersCompressionStatus(boolean is32BitJVM) { class Probe { @SuppressWarnings("unused") @@ -390,92 +282,6 @@ private static int msb(int value) { return 31 - Integer.numberOfLeadingZeros(value); } - interface AnonymousClassDefiner { - Class define(Class hostClass, byte[] data); - } - - /** - * Based on {@code MethodHandles.Lookup#defineHiddenClass}. - */ - static class MethodHandlesClassDefiner implements AnonymousClassDefiner { - private static Method defineMethod; - private static Object hiddenClassOptions; - private static Object lookupBase; - private static long lookupOffset; - - @Nullable - public static MethodHandlesClassDefiner newInstance() { - if (defineMethod == null) { - try { - Field trustedLookupField = MethodHandles.Lookup.class.getDeclaredField("IMPL_LOOKUP"); - lookupBase = UNSAFE.staticFieldBase(trustedLookupField); - lookupOffset = UNSAFE.staticFieldOffset(trustedLookupField); - hiddenClassOptions = hiddenClassOptions("NESTMATE"); - defineMethod = MethodHandles.Lookup.class - .getMethod("defineHiddenClass", byte[].class, boolean.class, hiddenClassOptions.getClass()); - } catch (ReflectiveOperationException e) { - return null; - } - } - return new MethodHandlesClassDefiner(); - } - - @Override - public Class define(Class hostClass, byte[] data) { - try { - MethodHandles.Lookup trustedLookup = (MethodHandles.Lookup) UNSAFE.getObject(lookupBase, lookupOffset); - MethodHandles.Lookup definedLookup = - (MethodHandles.Lookup) defineMethod.invoke(trustedLookup.in(hostClass), data, false, hiddenClassOptions); - return definedLookup.lookupClass(); - } catch (Exception e) { - e.printStackTrace(System.out); - return null; - } - } - - @SuppressWarnings("unchecked") - private static Object hiddenClassOptions(String... options) throws ClassNotFoundException { - @SuppressWarnings("rawtypes") - Class optionClass = Class.forName(MethodHandles.Lookup.class.getName() + "$ClassOption"); - Object classOptions = Array.newInstance(optionClass, options.length); - for (int i = 0; i < options.length; i++) { - Array.set(classOptions, i, Enum.valueOf(optionClass, options[i])); - } - return classOptions; - } - } - - /** - * Based on {@code Unsafe#defineAnonymousClass}. - */ - static class UnsafeClassDefiner implements AnonymousClassDefiner { - - private static Method defineMethod; - - @Nullable - public static UnsafeClassDefiner newInstance() { - if (defineMethod == null) { - try { - defineMethod = sun.misc.Unsafe.class - .getMethod("defineAnonymousClass", Class.class, byte[].class, Object[].class); - } catch (ReflectiveOperationException e) { - return null; - } - } - return new UnsafeClassDefiner(); - } - - @Override - public Class define(Class hostClass, byte[] data) { - try { - return (Class) defineMethod.invoke(UNSAFE, hostClass, data, null); - } catch (Exception e) { - e.printStackTrace(System.out); - return null; - } - } - } - static { try { Field theUnsafe = sun.misc.Unsafe.class.getDeclaredField("theUnsafe"); @@ -485,23 +291,11 @@ public Class define(Class hostClass, byte[] data) { BYTE_OFFSET = Unsafe.getUnsafe().arrayBaseOffset(byte[].class); BYTE_SCALE = msb(Unsafe.getUnsafe().arrayIndexScale(byte[].class)); - INT_OFFSET = Unsafe.getUnsafe().arrayBaseOffset(int[].class); - INT_SCALE = msb(Unsafe.getUnsafe().arrayIndexScale(int[].class)); - LONG_OFFSET = Unsafe.getUnsafe().arrayBaseOffset(long[].class); LONG_SCALE = msb(Unsafe.getUnsafe().arrayIndexScale(long[].class)); OVERRIDE = AccessibleObject_override_fieldOffset(); implAddExports = Module.class.getDeclaredMethod("implAddExports", String.class, Module.class); - - AnonymousClassDefiner classDefiner = UnsafeClassDefiner.newInstance(); - if (classDefiner == null) { - classDefiner = MethodHandlesClassDefiner.newInstance(); - } - if (classDefiner == null) { - throw new InstantiationException("failed to initialize class definer"); - } - anonymousClassDefiner = classDefiner; } catch (ReflectiveOperationException e) { throw new ExceptionInInitializerError(e); } @@ -510,17 +304,13 @@ public Class define(Class hostClass, byte[] data) { // A single allocation for all the off-heap native memory counters. // Might help with locality, given they're often incremented together. // All initial values set to 0. - final long nativeMemCountersArraySize = (6 + COUNTERS.length) * 8; + final long nativeMemCountersArraySize = (5 + COUNTERS.length) * 8; final long nativeMemCountersArray = UNSAFE.allocateMemory(nativeMemCountersArraySize); long ptr = nativeMemCountersArray; Vect.memset(nativeMemCountersArray, nativeMemCountersArraySize, 0); - // N.B.: The layout here is also used in `allocator.rs` for the Rust side. - // See: `struct MemTracking`. RSS_MEM_USED_ADDR = ptr; ptr += 8; - RSS_MEM_LIMIT_ADDR = ptr; - ptr += 8; MALLOC_COUNT_ADDR = ptr; ptr += 8; REALLOC_COUNT_ADDR = ptr; @@ -534,9 +324,5 @@ public Class define(Class hostClass, byte[] data) { NATIVE_MEM_COUNTER_ADDRS[i] = ptr; ptr += 8; } - for (int memoryTag = NATIVE_DEFAULT; memoryTag < MemoryTag.SIZE; ++memoryTag) { - NATIVE_ALLOCATORS[memoryTag - NATIVE_DEFAULT] = constructNativeAllocator( - nativeMemCountersArray, memoryTag); - } } } diff --git a/core/src/main/java/io/questdb/client/std/bytes/DirectByteSink.java b/core/src/main/java/io/questdb/client/std/bytes/DirectByteSink.java index a4b2465..7fbffcc 100644 --- a/core/src/main/java/io/questdb/client/std/bytes/DirectByteSink.java +++ b/core/src/main/java/io/questdb/client/std/bytes/DirectByteSink.java @@ -71,6 +71,7 @@ public long ptr() { return impl; } }; + public DirectByteSink(long initialCapacity, int memoryTag) { this(initialCapacity, memoryTag, false); } @@ -275,4 +276,4 @@ private void setImplPtr(long ptr) { static { Os.init(); } -} \ No newline at end of file +} diff --git a/core/src/main/java/io/questdb/client/std/ex/BytecodeException.java b/core/src/main/java/io/questdb/client/std/ex/BytecodeException.java deleted file mode 100644 index b3fea56..0000000 --- a/core/src/main/java/io/questdb/client/std/ex/BytecodeException.java +++ /dev/null @@ -1,33 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.std.ex; - -public class BytecodeException extends RuntimeException { - public static final BytecodeException INSTANCE = new BytecodeException(); - - private BytecodeException() { - super("Error in bytecode"); - } -} \ No newline at end of file diff --git a/core/src/main/java/io/questdb/client/std/str/DirectCharSequence.java b/core/src/main/java/io/questdb/client/std/str/DirectCharSequence.java deleted file mode 100644 index a3d14ee..0000000 --- a/core/src/main/java/io/questdb/client/std/str/DirectCharSequence.java +++ /dev/null @@ -1,33 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.std.str; - -import io.questdb.client.std.bytes.DirectSequence; - -/** - * A sequence of UTF-16 chars stored in native memory. - */ -public interface DirectCharSequence extends CharSequence, DirectSequence { -} \ No newline at end of file diff --git a/core/src/main/java/io/questdb/client/std/str/Utf8s.java b/core/src/main/java/io/questdb/client/std/str/Utf8s.java index 429b172..f87be3a 100644 --- a/core/src/main/java/io/questdb/client/std/str/Utf8s.java +++ b/core/src/main/java/io/questdb/client/std/str/Utf8s.java @@ -225,32 +225,6 @@ public static String stringFromUtf8Bytes(@NotNull Utf8Sequence seq) { return b.toString(); } - public static String stringFromUtf8BytesSafe(@NotNull Utf8Sequence seq) { - if (seq.size() == 0) { - return ""; - } - Utf16Sink b = getThreadLocalSink(); - utf8ToUtf16(seq, b); - return b.toString(); - } - - public static String toString(@Nullable Utf8Sequence s) { - return s == null ? null : s.toString(); - } - - public static String toString(@NotNull Utf8Sequence us, int start, int end, byte unescapeAscii) { - final Utf8Sink sink = getThreadLocalUtf8Sink(); - final int lastChar = end - 1; - for (int i = start; i < end; i++) { - byte b = us.byteAt(i); - sink.putAny(b); - if (b == unescapeAscii && i < lastChar && us.byteAt(i + 1) == unescapeAscii) { - i++; - } - } - return sink.toString(); - } - public static int utf8DecodeMultiByte(long lo, long hi, byte b, Utf16Sink sink) { if (b >> 5 == -2 && (b & 30) != 0) { return utf8Decode2Bytes(lo, hi, b, sink); @@ -329,28 +303,6 @@ public static boolean utf8ToUtf16(@NotNull Utf8Sequence seq, @NotNull Utf16Sink return utf8ToUtf16(seq, 0, seq.size(), sink); } - public static int validateUtf8(@NotNull Utf8Sequence seq) { - if (seq.isAscii()) { - return seq.size(); - } - int len = 0; - for (int i = 0, hi = seq.size(); i < hi; ) { - byte b = seq.byteAt(i); - if (b < 0) { - int n = validateUtf8MultiByte(seq, i, b); - if (n == -1) { - // UTF-8 error - return -1; - } - i += n; - } else { - ++i; - } - ++len; - } - return len; - } - /** * Returns up to 6 initial bytes of the given UTF-8 sequence (less if it's shorter) * packed into a zero-padded long value, in little-endian order. This prefix is @@ -674,61 +626,4 @@ private static int utf8DecodeMultiByte(Utf8Sequence seq, int index, byte b, @Not return utf8Decode4Bytes(seq, index, b, sink); } - private static int validateUtf8Decode2Bytes(@NotNull Utf8Sequence seq, int index) { - if (seq.size() - index < 2) { - return -1; - } - byte b2 = seq.byteAt(index + 1); - if (isNotContinuation(b2)) { - return -1; - } - return 2; - } - - private static int validateUtf8Decode3Bytes(@NotNull Utf8Sequence seq, int index, byte b1) { - if (seq.size() - index < 3) { - return -1; - } - byte b2 = seq.byteAt(index + 1); - byte b3 = seq.byteAt(index + 2); - - if (isMalformed3(b1, b2, b3)) { - return -1; - } - - char c = utf8ToChar(b1, b2, b3); - if (Character.isSurrogate(c)) { - return -1; - } - return 3; - } - - private static int validateUtf8Decode4Bytes(@NotNull Utf8Sequence seq, int index, int b) { - if (b >> 3 != -2 || seq.size() - index < 4) { - return -1; - } - byte b2 = seq.byteAt(index + 1); - byte b3 = seq.byteAt(index + 2); - byte b4 = seq.byteAt(index + 3); - - if (isMalformed4(b2, b3, b4)) { - return -1; - } - final int codePoint = getUtf8Codepoint(b, b2, b3, b4); - if (!Character.isSupplementaryCodePoint(codePoint)) { - return -1; - } - return 4; - } - - private static int validateUtf8MultiByte(Utf8Sequence seq, int index, byte b) { - if (b >> 5 == -2 && (b & 30) != 0) { - // we should allow 11000001, as it is a valid UTF8 byte? - return validateUtf8Decode2Bytes(seq, index); - } - if (b >> 4 == -2) { - return validateUtf8Decode3Bytes(seq, index, b); - } - return validateUtf8Decode4Bytes(seq, index, b); - } } \ No newline at end of file diff --git a/core/src/main/java/module-info.java b/core/src/main/java/module-info.java index 93d29fc..59e8343 100644 --- a/core/src/main/java/module-info.java +++ b/core/src/main/java/module-info.java @@ -56,4 +56,7 @@ exports io.questdb.client.cairo.arr; exports io.questdb.client.cutlass.line.array; exports io.questdb.client.cutlass.line.udp; + exports io.questdb.client.cutlass.qwp.client; + exports io.questdb.client.cutlass.qwp.protocol; + exports io.questdb.client.cutlass.qwp.websocket; } diff --git a/core/src/main/resources/io/questdb/client/bin/darwin-aarch64/libquestdb.dylib b/core/src/main/resources/io/questdb/client/bin/darwin-aarch64/libquestdb.dylib index 836ff3e..38f76b8 100644 Binary files a/core/src/main/resources/io/questdb/client/bin/darwin-aarch64/libquestdb.dylib and b/core/src/main/resources/io/questdb/client/bin/darwin-aarch64/libquestdb.dylib differ diff --git a/core/src/main/resources/io/questdb/client/bin/darwin-x86-64/libquestdb.dylib b/core/src/main/resources/io/questdb/client/bin/darwin-x86-64/libquestdb.dylib index bf0a00f..72d228a 100644 Binary files a/core/src/main/resources/io/questdb/client/bin/darwin-x86-64/libquestdb.dylib and b/core/src/main/resources/io/questdb/client/bin/darwin-x86-64/libquestdb.dylib differ diff --git a/core/src/main/resources/io/questdb/client/bin/linux-aarch64/libquestdb.so b/core/src/main/resources/io/questdb/client/bin/linux-aarch64/libquestdb.so index 4609bfd..76158e1 100644 Binary files a/core/src/main/resources/io/questdb/client/bin/linux-aarch64/libquestdb.so and b/core/src/main/resources/io/questdb/client/bin/linux-aarch64/libquestdb.so differ diff --git a/core/src/main/resources/io/questdb/client/bin/linux-x86-64/libquestdb.so b/core/src/main/resources/io/questdb/client/bin/linux-x86-64/libquestdb.so index 3c50a51..426586b 100644 Binary files a/core/src/main/resources/io/questdb/client/bin/linux-x86-64/libquestdb.so and b/core/src/main/resources/io/questdb/client/bin/linux-x86-64/libquestdb.so differ diff --git a/core/src/main/resources/io/questdb/client/bin/windows-x86-64/libquestdb.dll b/core/src/main/resources/io/questdb/client/bin/windows-x86-64/libquestdb.dll index 8df6af5..7aeb62f 100755 Binary files a/core/src/main/resources/io/questdb/client/bin/windows-x86-64/libquestdb.dll and b/core/src/main/resources/io/questdb/client/bin/windows-x86-64/libquestdb.dll differ diff --git a/core/src/test/java/io/questdb/client/test/AbstractQdbTest.java b/core/src/test/java/io/questdb/client/test/AbstractQdbTest.java deleted file mode 100644 index e0ef2e7..0000000 --- a/core/src/test/java/io/questdb/client/test/AbstractQdbTest.java +++ /dev/null @@ -1,675 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test; - -import io.questdb.client.std.Numbers; -import io.questdb.client.std.str.StringSink; -import io.questdb.client.std.str.Utf16Sink; -import io.questdb.client.test.tools.TestUtils; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Assume; -import org.junit.Before; -import org.junit.BeforeClass; - -import java.io.IOException; -import java.io.InputStream; -import java.sql.Array; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.JDBCType; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.sql.Statement; -import java.sql.Timestamp; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeFormatterBuilder; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicLong; - -import static io.questdb.client.std.Numbers.hexDigits; -import static io.questdb.client.test.tools.TestUtils.assertEventually; -import static java.time.temporal.ChronoField.*; - -public class AbstractQdbTest extends AbstractTest { - - protected static final StringSink sink = new StringSink(); - private static final DateTimeFormatter DATE_TIME_FORMATTER; - // Configuration defaults - private static final String DEFAULT_HOST = "127.0.0.1"; - private static final int DEFAULT_HTTP_PORT = 9000; - private static final String DEFAULT_PG_PASSWORD = "quest"; - private static final int DEFAULT_PG_PORT = 8812; - private static final String DEFAULT_PG_USER = "admin"; - // Table name counter for uniqueness - private static final AtomicLong TABLE_NAME_COUNTER = new AtomicLong(System.currentTimeMillis()); - // Shared PostgreSQL connection - private static Connection pgConnection; - - /** - * Print the output of a SQL query to TSV format. - */ - public static long printToSink(StringSink sink, ResultSet rs) throws SQLException { - // dump metadata - ResultSetMetaData metaData = rs.getMetaData(); - final int columnCount = metaData.getColumnCount(); - for (int i = 0; i < columnCount; i++) { - if (i > 0) { - sink.put('\t'); - } - - sink.put(metaData.getColumnName(i + 1)); - } - sink.put('\n'); - - Timestamp timestamp; - long rows = 0; - while (rs.next()) { - rows++; - for (int i = 1; i <= columnCount; i++) { - if (i > 1) { - sink.put('\t'); - } - switch (JDBCType.valueOf(metaData.getColumnType(i))) { - case VARCHAR: - case NUMERIC: - String stringValue = rs.getString(i); - if (rs.wasNull()) { - sink.put("null"); - } else { - sink.put(stringValue); - } - break; - case INTEGER: - int intValue = rs.getInt(i); - if (rs.wasNull()) { - sink.put("null"); - } else { - sink.put(intValue); - } - break; - case DOUBLE: - double doubleValue = rs.getDouble(i); - if (rs.wasNull()) { - sink.put("null"); - } else { - sink.put(doubleValue); - } - break; - case TIMESTAMP: - timestamp = rs.getTimestamp(i); - if (timestamp == null) { - sink.put("null"); - } else { - sink.put(DATE_TIME_FORMATTER.format(timestamp.toLocalDateTime())); - } - break; - case REAL: - float floatValue = rs.getFloat(i); - if (rs.wasNull()) { - sink.put("null"); - } else { - sink.put(floatValue); - } - break; - case SMALLINT: - sink.put(rs.getShort(i)); - break; - case BIGINT: - long longValue = rs.getLong(i); - if (rs.wasNull()) { - sink.put("null"); - } else { - sink.put(longValue); - } - break; - case CHAR: - String strValue = rs.getString(i); - if (rs.wasNull()) { - sink.put("null"); - } else { - sink.put(strValue.charAt(0)); - } - break; - case BIT: - sink.put(rs.getBoolean(i)); - break; - case TIME: - case DATE: - timestamp = rs.getTimestamp(i); - if (rs.wasNull()) { - sink.put("null"); - } else { - sink.put(DATE_TIME_FORMATTER.format(timestamp.toLocalDateTime())); - } - break; - case BINARY: - InputStream stream = rs.getBinaryStream(i); - if (rs.wasNull()) { - sink.put("null"); - } else { - toSink(stream, sink); - } - break; - case OTHER: - Object object = rs.getObject(i); - if (rs.wasNull()) { - sink.put("null"); - } else { - sink.put(object.toString()); - } - break; - case ARRAY: - Array array = rs.getArray(i); - if (array == null) { - sink.put("null"); - } else { - writeArrayContent(sink, array.getArray()); - } - break; - default: - assert false; - } - } - sink.put('\n'); - } - return rows; - } - - @BeforeClass - public static void setUpStatic() { - AbstractTest.setUpStatic(); - if (getQuestDBRunning()) { - System.err.printf("CLEANING UP TEST TABLES%n"); - // Cleanup all test tables before starting tests - try (Connection conn = getPgConnection(); - Statement readStmt = conn.createStatement(); - Statement stmt = conn.createStatement(); - ResultSet rs = readStmt - .executeQuery("SELECT table_name FROM tables() WHERE table_name LIKE 'test_%'")) { - while (rs.next()) { - String tableName = rs.getString(1); - try { - stmt.execute(String.format("DROP TABLE IF EXISTS '%s'", tableName)); - LOG.info("Dropped test table: {}", tableName); - } catch (SQLException e) { - LOG.warn("Failed to drop test table {}: {}", tableName, e.getMessage()); - } - } - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - } - - @AfterClass - public static void tearDownStatic() { - closePgConnection(); - } - - @Before - public void setUp() { - super.setUp(); - } - - @After - public void tearDown() throws Exception { - super.tearDown(); - } - - private static void toSink(InputStream is, Utf16Sink sink) { - // limit what we print - byte[] bb = new byte[1]; - int i = 0; - try { - while (is.read(bb) > 0) { - byte b = bb[0]; - if (i > 0) { - if ((i % 16) == 0) { - sink.put('\n'); - Numbers.appendHexPadded(sink, i); - } - } else { - Numbers.appendHexPadded(sink, i); - } - sink.putAscii(' '); - - final int v; - if (b < 0) { - v = 256 + b; - } else { - v = b; - } - - if (v < 0x10) { - sink.putAscii('0'); - sink.putAscii(hexDigits[b]); - } else { - sink.putAscii(hexDigits[v / 0x10]); - sink.putAscii(hexDigits[v % 0x10]); - } - - i++; - } - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private static void writeArrayContent(StringSink sink, Object array) { - if (array == null) { - sink.put("null"); - return; - } - if (!array.getClass().isArray()) { - if (array instanceof Number) { - if (array instanceof Double) { - double d = ((Number) array).doubleValue(); - if (Numbers.isNull(d)) { - sink.put("null"); - } else { - sink.put(d); - } - } - if (array instanceof Float) { - float f = ((Number) array).floatValue(); - if (Numbers.isNull(f)) { - sink.put("null"); - } else { - sink.put(f); - } - } - if (array instanceof Long) { - long l = ((Number) array).longValue(); - if (l == Numbers.LONG_NULL) { - sink.put("null"); - } else { - sink.put(l); - } - } - } else if (array instanceof Boolean) { - sink.put((Boolean) array); - } else { - sink.put(array.toString()); - } - return; - } - - sink.put('{'); - int length = java.lang.reflect.Array.getLength(array); - for (int i = 0; i < length; i++) { - Object element = java.lang.reflect.Array.get(array, i); - writeArrayContent(sink, element); - - if (i < length - 1) { - sink.put(','); - } - } - sink.put('}'); - } - - /** - * Close the shared PostgreSQL connection. - */ - protected static synchronized void closePgConnection() { - if (pgConnection != null) { - try { - pgConnection.close(); - LOG.info("Closed PostgreSQL connection"); - } catch (SQLException e) { - LOG.warn("Error closing PostgreSQL connection", e); - } finally { - pgConnection = null; - } - } - } - - /** - * Get configuration value from environment variable or system property. - * Environment variables take precedence over system properties. - */ - protected static String getConfig(String envKey, String sysPropKey, String defaultValue) { - String value = System.getenv(envKey); - if (value != null && !value.isEmpty()) { - return value; - } - return System.getProperty(sysPropKey, defaultValue); - } - - protected static boolean getConfigBool(String envKey, String sysPropKey, boolean defaultValue) { - String value = getConfig(envKey, sysPropKey, null); - if (value != null) { - return Boolean.parseBoolean(value); - } - return defaultValue; - } - - protected static int getConfigInt(String envKey, String sysPropKey, int defaultValue) { - String value = getConfig(envKey, sysPropKey, null); - if (value != null) { - try { - return Integer.parseInt(value); - } catch (NumberFormatException e) { - LOG.warn("Invalid integer value for {}/{}: {}, using default: {}", - envKey, sysPropKey, value, defaultValue); - } - } - return defaultValue; - } - - /** - * Get HTTP port. - */ - protected static int getHttpPort() { - return getConfigInt("QUESTDB_HTTP_PORT", "questdb.http.port", DEFAULT_HTTP_PORT); - } - - /** - * Get or create the shared PostgreSQL connection. - */ - protected static synchronized Connection getPgConnection() throws SQLException { - if (pgConnection == null || pgConnection.isClosed()) { - pgConnection = initPgConnection(); - } - return pgConnection; - } - - /** - * Get PostgreSQL password. - */ - protected static String getPgPassword() { - return getConfig("QUESTDB_PG_PASSWORD", "questdb.pg.password", DEFAULT_PG_PASSWORD); - } - - /** - * Get PostgreSQL wire protocol port. - */ - protected static int getPgPort() { - return getConfigInt("QUESTDB_PG_PORT", "questdb.pg.port", DEFAULT_PG_PORT); - } - - /** - * Get PostgreSQL user. - */ - protected static String getPgUser() { - return getConfig("QUESTDB_PG_USER", "questdb.pg.user", DEFAULT_PG_USER); - } - - /** - * Get whether a QuestDB instance is running locally. - */ - protected static boolean getQuestDBRunning() { - return getConfigBool("QUESTDB_RUNNING", "questdb.running", false); - } - - /** - * Get QuestDB host address. - */ - protected static String getQuestDbHost() { - return getConfig("QUESTDB_HOST", "questdb.host", DEFAULT_HOST); - } - - /** - * Initialize a new PostgreSQL connection to QuestDB. - */ - protected static Connection initPgConnection() throws SQLException { - String host = getQuestDbHost(); - int port = getPgPort(); - String user = getPgUser(); - String password = getPgPassword(); - - String url = String.format("jdbc:postgresql://%s:%d/qdb?sslmode=disable", host, port); - LOG.info("Connecting to QuestDB via PostgreSQL wire protocol: {}", url); - - return DriverManager.getConnection(url, user, password); - } - - /** - * Assert that SQL query results match expected TSV, polling until they do or - * timeout is reached. - */ - protected void assertSqlEventually(CharSequence expected, String sql) throws Exception { - assertEventually(() -> { - try (Statement statement = getPgConnection().createStatement(); - ResultSet rs = statement.executeQuery(sql)) { - sink.clear(); - printToSink(sink, rs); - TestUtils.assertEquals(expected, sink); - } catch (SQLException e) { - throw new RuntimeException(e); - } - }, 5); - } - - /** - * Assert that table exists, polling until it does or timeout is reached. - */ - protected void assertTableExistsEventually(CharSequence tableName) throws Exception { - assertEventually(() -> { - try (Statement stmt = getPgConnection().createStatement(); - ResultSet rs = stmt.executeQuery( - String.format("SELECT COUNT(*) AS cnt FROM tables() WHERE table_name = '%s'", tableName))) { - Assert.assertTrue(rs.next()); - final long actualSize = rs.getLong(1); - Assert.assertEquals(1, actualSize); - } catch (SQLException e) { - throw new RuntimeException(e); - } - }, 5); - } - - /** - * Assert that table has expected size, polling until it does or timeout is - * reached. - */ - protected void assertTableSizeEventually(CharSequence tableName, int expectedSize) throws Exception { - final String sql = String.format("SELECT COUNT(*) AS cnt FROM \"%s\"", tableName); - assertEventually(() -> { - try ( - Statement stmt = getPgConnection().createStatement(); - ResultSet rs = stmt.executeQuery(sql)) { - Assert.assertTrue(rs.next()); - final long actualSize = rs.getLong(1); - Assert.assertEquals(expectedSize, actualSize); - } catch (SQLException e) { - // If the table does not exist yet, we may get an exception - if (e.getMessage().contains("table does not exist")) { - Assert.fail("Table not found: " + tableName); - } - throw new RuntimeException(e); - } - }, 15); - } - - /** - * Drop a table if it exists. - */ - protected void dropTable(String tableName) { - try { - executeSql(String.format("DROP TABLE IF EXISTS '%s'", tableName)); - LOG.info("Dropped table: {}", tableName); - } catch (SQLException e) { - LOG.warn("Failed to drop table {}: {}", tableName, e.getMessage()); - } - } - - /** - * Drop multiple tables. - */ - protected void dropTables(List tableNames) { - for (String tableName : tableNames) { - dropTable(tableName); - } - } - - /** - * Execute SQL and assert no exceptions are thrown. - */ - protected void execute(String sql) throws Exception { - try (Statement statement = getPgConnection().createStatement()) { - statement.execute(sql); - } - } - - /** - * Execute a SELECT query and return results as a list of maps. - * Each map represents a row with column names as keys. - */ - protected List> executeQuery(String sql) throws SQLException { - List> results = new ArrayList<>(); - - try (Statement stmt = getPgConnection().createStatement(); - ResultSet rs = stmt.executeQuery(sql)) { - - ResultSetMetaData metaData = rs.getMetaData(); - int columnCount = metaData.getColumnCount(); - - while (rs.next()) { - Map row = new LinkedHashMap<>(); - for (int i = 1; i <= columnCount; i++) { - String columnName = metaData.getColumnName(i); - Object value = rs.getObject(i); - row.put(columnName, value); - } - results.add(row); - } - } - - return results; - } - - /** - * Execute a DDL or DML statement (CREATE, DROP, INSERT, etc.). - */ - protected void executeSql(String sql) throws SQLException { - try (Statement stmt = getPgConnection().createStatement()) { - stmt.execute(sql); - } - } - - /** - * Generate a unique table name with the given prefix. - * This ensures test isolation when running tests in parallel. - */ - protected String generateTableName(String prefix) { - return prefix + "_" + TABLE_NAME_COUNTER.incrementAndGet(); - } - - /** - * Query table contents with optional ORDER BY clause and return as - * TSV-formatted string. - */ - protected String queryTableAsTsv(String tableName, String orderBy) throws SQLException { - StringBuilder sb = new StringBuilder(); - - String sql = String.format("SELECT * FROM '%s'", tableName); - if (orderBy != null && !orderBy.isEmpty()) { - sql += " ORDER BY " + orderBy; - } - - try (Statement stmt = getPgConnection().createStatement(); - ResultSet rs = stmt.executeQuery(sql)) { - - ResultSetMetaData metaData = rs.getMetaData(); - int columnCount = metaData.getColumnCount(); - - // Header row - for (int i = 1; i <= columnCount; i++) { - if (i > 1) { - sb.append('\t'); - } - sb.append(metaData.getColumnName(i)); - } - sb.append('\n'); - - // Data rows - while (rs.next()) { - for (int i = 1; i <= columnCount; i++) { - if (i > 1) { - sb.append('\t'); - } - Object value = rs.getObject(i); - sb.append(value == null ? "" : value.toString()); - } - sb.append('\n'); - } - } - - return sb.toString(); - } - - /** - * Check if table exists (non-blocking, no polling). - */ - protected boolean tableExists(String tableName) throws SQLException { - List> result = executeQuery( - String.format("SELECT table_name FROM tables() WHERE table_name = '%s'", tableName)); - return !result.isEmpty(); - } - - /** - * Track a table for cleanup after the test. - * If the table already exists, we block until it is dropped. - * - * @param tableName the name of the table to track - */ - protected void useTable(String tableName) throws Exception { - TestUtils.assertEventually(() -> { - try { - if (!tableExists(tableName)) { - return; - } - Thread.sleep(100); - dropTable(tableName); - Assert.fail("Table " + tableName + " already exists. Dropping it."); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException("Interrupted while waiting for table to be dropped: " + tableName, e); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - } - - static { - DATE_TIME_FORMATTER = new DateTimeFormatterBuilder() - .parseCaseInsensitive() - .append(DateTimeFormatter.ISO_LOCAL_DATE) - .appendLiteral('T') - .appendValue(HOUR_OF_DAY, 2) - .appendLiteral(':') - .appendValue(MINUTE_OF_HOUR, 2) - .appendLiteral(':') - .appendValue(SECOND_OF_MINUTE, 2) - .appendFraction(NANO_OF_SECOND, 9, 9, true) - .appendLiteral('Z') - .toFormatter(); - } -} diff --git a/core/src/test/java/io/questdb/client/test/LineSenderBuilderTest.java b/core/src/test/java/io/questdb/client/test/LineSenderBuilderTest.java deleted file mode 100644 index 5eb47fe..0000000 --- a/core/src/test/java/io/questdb/client/test/LineSenderBuilderTest.java +++ /dev/null @@ -1,799 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test; - -import io.questdb.client.Sender; -import io.questdb.client.cutlass.line.LineSenderException; -import io.questdb.client.test.tools.TestUtils; -import org.junit.Assert; -import org.junit.Test; - -import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; -import static org.junit.Assert.fail; - -/** - * Unit tests for LineSenderBuilder that don't require a running QuestDB instance. - * Tests that require an actual QuestDB connection have been moved to integration tests. - */ -public class LineSenderBuilderTest { - private static final String AUTH_TOKEN_KEY1 = "UvuVb1USHGRRT08gEnwN2zGZrvM4MsLQ5brgF6SVkAw="; - private static final String LOCALHOST = "localhost"; - private static final char[] TRUSTSTORE_PASSWORD = "questdb".toCharArray(); - private static final String TRUSTSTORE_PATH = "/keystore/server.keystore"; - - @Test - public void testAddressDoubleSet_firstAddressThenAddress() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).address(LOCALHOST); - try { - builder.address("127.0.0.1"); - builder.build(); - fail("should not allow double host set"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "mismatch"); - } - }); - } - - @Test - public void testAddressEmpty() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP); - try { - builder.address(""); - fail("empty address should fail"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "address cannot be empty"); - } - }); - } - - @Test - public void testAddressEndsWithColon() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP); - try { - builder.address("foo:"); - fail("should fail when address ends with colon"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "invalid address"); - } - }); - } - - @Test - public void testAddressNull() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP); - try { - builder.address(null); - fail("null address should fail"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "null"); - } - }); - } - - @Test - public void testAuthDoubleSet() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).enableAuth("foo").authToken(AUTH_TOKEN_KEY1); - try { - builder.enableAuth("bar"); - fail("should not allow double auth set"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "already configured"); - } - }); - } - - @Test - public void testAuthTooSmallBuffer() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP) - .enableAuth("foo").authToken(AUTH_TOKEN_KEY1).address(LOCALHOST + ":9001") - .bufferCapacity(1); - builder.build(); - fail("tiny buffer should NOT be allowed as it wont fit auth challenge"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "minimalCapacity"); - TestUtils.assertContains(e.getMessage(), "requestedCapacity"); - } - }); - } - - @Test - public void testAuthWithBadToken() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder.AuthBuilder builder = Sender.builder(Sender.Transport.TCP).enableAuth("foo"); - try { - builder.authToken("bar token"); - fail("bad token should not be imported"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "could not import token"); - } - }); - } - - @Test - public void testAutoFlushIntervalMustBePositive() { - try (Sender ignored = Sender.builder(Sender.Transport.HTTP).autoFlushIntervalMillis(0).build()) { - fail("auto-flush must be positive"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "auto flush interval cannot be negative [autoFlushIntervalMillis=0]"); - } - - try (Sender ignored = Sender.builder(Sender.Transport.HTTP).autoFlushIntervalMillis(-1).build()) { - fail("auto-flush must be positive"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "auto flush interval cannot be negative [autoFlushIntervalMillis=-1]"); - } - } - - @Test - public void testAutoFlushIntervalNotSupportedForTcp() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.TCP).address(LOCALHOST).autoFlushIntervalMillis(1).build(); - fail("auto flush interval should not be supported for TCP"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "auto flush interval is not supported for TCP protocol"); - } - }); - } - - @Test - public void testAutoFlushInterval_afterAutoFlushDisabled() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.HTTP).disableAutoFlush().autoFlushIntervalMillis(1); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "cannot set auto flush interval when interval based auto-flush is already disabled"); - } - }); - } - - @Test - public void testAutoFlushInterval_doubleConfiguration() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.HTTP).autoFlushIntervalMillis(1).autoFlushIntervalMillis(1); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "auto flush interval was already configured [autoFlushIntervalMillis=1]"); - } - }); - } - - @Test - public void testAutoFlushRowsCannotBeNegative() { - try (Sender ignored = Sender.builder(Sender.Transport.HTTP).autoFlushRows(-1).build()) { - fail("auto-flush must be positive"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "auto flush rows cannot be negative [autoFlushRows=-1]"); - } - } - - @Test - public void testAutoFlushRowsNotSupportedForTcp() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.TCP).address(LOCALHOST).autoFlushRows(1).build(); - fail("auto flush rows should not be supported for TCP"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "auto flush rows is not supported for TCP protocol"); - } - }); - } - - @Test - public void testAutoFlushRows_doubleConfiguration() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.HTTP).autoFlushRows(1).autoFlushRows(1); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "auto flush rows was already configured [autoFlushRows=1]"); - } - }); - } - - @Test - public void testBufferSizeDoubleSet() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).bufferCapacity(1024); - try { - builder.bufferCapacity(1024); - fail("should not allow double buffer capacity set"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "already configured"); - } - }); - } - - @Test - public void testConfStringValidation() throws Exception { - assertMemoryLeak(() -> { - assertConfStrError("foo", "invalid schema [schema=foo, supported-schemas=[http, https, tcp, tcps]]"); - assertConfStrError("badschema::addr=bar;", "invalid schema [schema=badschema, supported-schemas=[http, https, tcp, tcps]]"); - assertConfStrError("http::addr=localhost:-1;", "invalid port [port=-1]"); - assertConfStrError("http::auto_flush=on;", "addr is missing"); - assertConfStrError("http::addr=localhost;tls_roots=/some/path;", "tls_roots was configured, but tls_roots_password is missing"); - assertConfStrError("http::addr=localhost;tls_roots_password=hunter123;", "tls_roots_password was configured, but tls_roots is missing"); - assertConfStrError("tcp::addr=localhost;user=foo;", "token cannot be empty nor null"); - assertConfStrError("tcp::addr=localhost;username=foo;", "token cannot be empty nor null"); - assertConfStrError("tcp::addr=localhost;token=foo;", "TCP token is configured, but user is missing"); - assertConfStrError("http::addr=localhost;user=foo;", "password cannot be empty nor null"); - assertConfStrError("http::addr=localhost;username=foo;", "password cannot be empty nor null"); - assertConfStrError("http::addr=localhost;pass=foo;", "HTTP password is configured, but username is missing"); - assertConfStrError("http::addr=localhost;password=foo;", "HTTP password is configured, but username is missing"); - assertConfStrError("tcp::addr=localhost;pass=foo;", "password is not supported for TCP protocol"); - assertConfStrError("tcp::addr=localhost;password=foo;", "password is not supported for TCP protocol"); - assertConfStrError("tcp::addr=localhost;retry_timeout=;", "retry_timeout cannot be empty"); - assertConfStrError("tcp::addr=localhost;max_buf_size=;", "max_buf_size cannot be empty"); - assertConfStrError("tcp::addr=localhost;init_buf_size=;", "init_buf_size cannot be empty"); - assertConfStrError("http::addr=localhost:8080;tls_verify=unsafe_off;", "TLS validation disabled, but TLS was not enabled"); - assertConfStrError("http::addr=localhost:8080;tls_verify=bad;", "invalid tls_verify [value=bad, allowed-values=[on, unsafe_off]]"); - assertConfStrError("tcps::addr=localhost;pass=unsafe_off;", "password is not supported for TCP protocol"); - assertConfStrError("tcps::addr=localhost;password=unsafe_off;", "password is not supported for TCP protocol"); - assertConfStrError("http::addr=localhost:8080;max_buf_size=-32;", "maximum buffer capacity cannot be less than initial buffer capacity [maximumBufferCapacity=-32, initialBufferCapacity=65536]"); - assertConfStrError("http::addr=localhost:8080;max_buf_size=notanumber;", "invalid max_buf_size [value=notanumber]"); - assertConfStrError("http::addr=localhost:8080;init_buf_size=notanumber;", "invalid init_buf_size [value=notanumber]"); - assertConfStrError("http::addr=localhost:8080;init_buf_size=-42;", "buffer capacity cannot be negative [capacity=-42]"); - assertConfStrError("http::addr=localhost:8080;auto_flush_rows=0;", "invalid auto_flush_rows [value=0]"); - assertConfStrError("http::addr=localhost:8080;auto_flush_rows=notanumber;", "invalid auto_flush_rows [value=notanumber]"); - assertConfStrError("http::addr=localhost:8080;auto_flush=invalid;", "invalid auto_flush [value=invalid, allowed-values=[on, off]]"); - assertConfStrError("http::addr=localhost:8080;auto_flush=off;auto_flush_rows=100;", "cannot set auto flush rows when auto-flush is already disabled"); - assertConfStrError("http::addr=localhost:8080;auto_flush_rows=100;auto_flush=off;", "auto flush rows was already configured [autoFlushRows=100]"); - assertConfStrError("HTTP::addr=localhost;", "invalid schema [schema=HTTP, supported-schemas=[http, https, tcp, tcps]]"); - assertConfStrError("HTTPS::addr=localhost;", "invalid schema [schema=HTTPS, supported-schemas=[http, https, tcp, tcps]]"); - assertConfStrError("TCP::addr=localhost;", "invalid schema [schema=TCP, supported-schemas=[http, https, tcp, tcps]]"); - assertConfStrError("TCPS::addr=localhost;", "invalid schema [schema=TCPS, supported-schemas=[http, https, tcp, tcps]]"); - assertConfStrError("http::addr=localhost;auto_flush=off;auto_flush_interval=1;", "cannot set auto flush interval when interval based auto-flush is already disabled"); - assertConfStrError("http::addr=localhost;auto_flush=off;auto_flush_rows=1;", "cannot set auto flush rows when auto-flush is already disabled"); - assertConfStrError("http::addr=localhost;auto_flush_bytes=1024;", "auto_flush_bytes is only supported for TCP transport"); - assertConfStrError("http::addr=localhost;protocol_version=10", "current client only supports protocol version 1(text format for all datatypes), 2(binary format for part datatypes), 3(decimal datatype) or explicitly unset"); - assertConfStrError("http::addr=localhost:48884;max_name_len=10;", "max_name_len must be at least 16 bytes [max_name_len=10]"); - - assertConfStrOk("addr=localhost:8080", "auto_flush_rows=100", "protocol_version=1"); - assertConfStrOk("addr=localhost:8080", "auto_flush=on", "auto_flush_rows=100", "protocol_version=2"); - assertConfStrOk("addr=localhost:8080", "auto_flush_rows=100", "auto_flush=on", "protocol_version=2"); - assertConfStrOk("addr=localhost", "auto_flush=on", "protocol_version=2"); - assertConfStrOk("addr=localhost:8080", "max_name_len=1024", "protocol_version=2"); - - assertConfStrError("tcp::addr=localhost;auto_flush_bytes=1024;init_buf_size=2048;", "TCP transport requires init_buf_size and auto_flush_bytes to be set to the same value [init_buf_size=2048, auto_flush_bytes=1024]"); - assertConfStrError("tcp::addr=localhost;init_buf_size=1024;auto_flush_bytes=2048;", "TCP transport requires init_buf_size and auto_flush_bytes to be set to the same value [init_buf_size=1024, auto_flush_bytes=2048]"); - assertConfStrError("tcp::addr=localhost;auto_flush_bytes=off;", "TCP transport must have auto_flush_bytes enabled"); - - assertConfStrOk("http::addr=localhost;auto_flush=off;protocol_version=2;"); - assertConfStrOk("http::addr=localhost;protocol_version=2;"); - assertConfStrOk("http::addr=localhost;auto_flush_interval=off;protocol_version=2;"); - assertConfStrOk("http::addr=localhost;auto_flush_rows=off;protocol_version=2;"); - assertConfStrOk("http::addr=localhost;auto_flush_interval=off;auto_flush_rows=off;protocol_version=1;"); - assertConfStrOk("http::addr=localhost;auto_flush_interval=off;auto_flush_rows=1;protocol_version=2;"); - assertConfStrOk("http::addr=localhost;auto_flush_rows=off;auto_flush_interval=1;protocol_version=2;"); - assertConfStrOk("http::addr=localhost;auto_flush_interval=off;auto_flush_rows=off;auto_flush=off;protocol_version=2;"); - assertConfStrOk("http::addr=localhost;auto_flush=off;auto_flush_interval=off;auto_flush_rows=off;protocol_version=1;"); - assertConfStrOk("http::addr=localhost:8080;protocol_version=2;"); - assertConfStrOk("http::addr=localhost:8080;token=foo;protocol_version=2;"); - assertConfStrOk("http::addr=localhost:8080;token=foo=bar;protocol_version=2;"); - assertConfStrOk("addr=localhost:8080", "token=foo", "retry_timeout=1000", "max_buf_size=1000000", "protocol_version=2"); - assertConfStrOk("addr=localhost:8080", "token=foo", "retry_timeout=1000", "max_buf_size=1000000", "protocol_version=1"); - assertConfStrOk("http::addr=localhost:8080;token=foo;max_buf_size=1000000;retry_timeout=1000;protocol_version=2;"); - assertConfStrOk("https::addr=localhost:8080;tls_verify=unsafe_off;auto_flush_rows=100;protocol_version=2;"); - assertConfStrOk("https::addr=localhost:8080;tls_verify=unsafe_off;auto_flush_rows=100;protocol_version=2;max_name_len=256;"); - assertConfStrOk("https::addr=localhost:8080;tls_verify=on;protocol_version=2;"); - assertConfStrOk("https::addr=localhost:8080;tls_verify=on;protocol_version=3;"); - assertConfStrError("https::addr=2001:0db8:85a3:0000:0000:8a2e:0370:7334;tls_verify=on;", "cannot parse a port from the address, use IPv4 address or a domain name [address=2001:0db8:85a3:0000:0000:8a2e:0370:7334]"); - assertConfStrError("https::addr=[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:9000;tls_verify=on;", "cannot parse a port from the address, use IPv4 address or a domain name [address=[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:9000]"); - }); - } - - @Test - public void testCustomTruststoreButTlsNotEnabled() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP) - .advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD) - .address(LOCALHOST); - try { - builder.build(); - fail("should fail when custom trust store configured, but TLS not enabled"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "TLS was not enabled"); - } - }); - } - - @Test - public void testCustomTruststoreDoubleSet() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD); - try { - builder.advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD); - fail("should not allow double custom trust store set"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "already configured"); - } - }); - } - - @Test - public void testCustomTruststorePasswordCannotBeNull() { - try { - Sender.builder(Sender.Transport.TCP).advancedTls().customTrustStore(TRUSTSTORE_PATH, null); - fail("should not allow null trust store password"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "trust store password cannot be null"); - } - } - - @Test - public void testCustomTruststorePathCannotBeBlank() { - try { - Sender.builder(Sender.Transport.TCP).advancedTls().customTrustStore("", TRUSTSTORE_PASSWORD); - fail("should not allow blank trust store path"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "trust store path cannot be empty nor null"); - } - - try { - Sender.builder(Sender.Transport.TCP).advancedTls().customTrustStore(null, TRUSTSTORE_PASSWORD); - fail("should not allow null trust store path"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "trust store path cannot be empty nor null"); - } - } - - @Test - public void testDisableAutoFlushNotSupportedForTcp() throws Exception { - assertMemoryLeak(() -> { - try (Sender ignored = Sender.builder(Sender.Transport.TCP).address(LOCALHOST).disableAutoFlush().build()) { - fail("TCP does not support disabling auto-flush"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "auto-flush is not supported for TCP protocol"); - } - }); - } - - @Test - public void testDnsResolutionFail() throws Exception { - assertMemoryLeak(() -> { - try (Sender ignored = Sender.builder(Sender.Transport.TCP).address("this-domain-does-not-exist-i-hope-better-to-use-a-silly-tld.silly-tld").build()) { - fail("dns resolution errors should fail fast"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "could not resolve"); - } - }); - } - - @Test - public void testDuplicatedAddresses() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.TCP).address("localhost:9000").address("localhost:9000"); - Assert.fail("should not allow multiple addresses"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "duplicated addresses are not allowed [address=localhost:9000]"); - } - - try { - Sender.builder(Sender.Transport.TCP).address("localhost:9000").address("localhost:9001").address("localhost:9000"); - Assert.fail("should not allow multiple addresses"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "duplicated addresses are not allowed [address=localhost:9000]"); - } - }); - } - - @Test - public void testDuplicatedAddressesWithDifferentPortsAllowed() throws Exception { - assertMemoryLeak(() -> { - Sender.builder(Sender.Transport.TCP).address("localhost:9000").address("localhost:9001"); - }); - } - - @Test - public void testDuplicatedAddressesWithNoPortsAllowed() throws Exception { - assertMemoryLeak(() -> { - Sender.builder(Sender.Transport.TCP).address("localhost:9000").address("localhost"); - Sender.builder(Sender.Transport.TCP).address("localhost").address("localhost:9000"); - }); - } - - @Test - public void testFailFastWhenSetCustomTrustStoreTwice() { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD); - try { - builder.advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD); - fail("should not allow double custom trust store set"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "already configured"); - } - } - - @Test - public void testFirstTlsValidationDisabledThenCustomTruststore() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP) - .advancedTls().disableCertificateValidation(); - try { - builder.advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD); - fail("should not allow custom truststore when TLS validation was disabled disabled"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "TLS validation was already disabled"); - } - }); - } - - @Test - public void testHostNorAddressSet() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP); - try { - builder.build(); - fail("not host should fail"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "server address not set"); - } - }); - } - - @Test - public void testHttpTokenNotSupportedForTcp() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.TCP).address(LOCALHOST).httpToken("foo").build(); - fail("HTTP token should not be supported for TCP"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "HTTP token authentication is not supported for TCP protocol"); - } - }); - } - - @Test - public void testInvalidHttpTimeout() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.HTTP).address("someurl").httpTimeoutMillis(0); - fail("should fail with bad http time"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "HTTP timeout must be positive [timeout=0]"); - } - - try { - Sender.builder(Sender.Transport.HTTP).address("someurl").httpTimeoutMillis(-1); - fail("should fail with bad http time"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "HTTP timeout must be positive [timeout=-1]"); - } - - try { - Sender.builder(Sender.Transport.HTTP).address("someurl").httpTimeoutMillis(100).httpTimeoutMillis(200); - fail("should fail with bad http time"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "HTTP timeout was already configured [timeout=100]"); - } - - try { - Sender.builder(Sender.Transport.TCP).address("localhost").httpTimeoutMillis(5000).build(); - fail("should fail with bad http time"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "HTTP timeout is not supported for TCP protocol"); - } - }); - } - - @Test - public void testInvalidRetryTimeout() { - try { - Sender.builder(Sender.Transport.HTTP).retryTimeoutMillis(-1); - Assert.fail(); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "retry timeout cannot be negative [retryTimeoutMillis=-1]"); - } - - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.HTTP).retryTimeoutMillis(100); - try { - builder.retryTimeoutMillis(200); - Assert.fail(); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "retry timeout was already configured [retryTimeoutMillis=100]"); - } - } - - @Test - public void testMalformedPortInAddress() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP); - try { - builder.address("foo:nonsense12334"); - fail("should fail with malformated port"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "cannot parse a port from the address"); - } - }); - } - - @Test - public void testMaxRequestBufferSizeCannotBeLessThanDefault() throws Exception { - assertMemoryLeak(() -> { - try (Sender ignored = Sender.builder(Sender.Transport.HTTP) - .address("localhost:1") - .maxBufferCapacity(65535) - .build() - ) { - Assert.fail(); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "maximum buffer capacity cannot be less than initial buffer capacity [maximumBufferCapacity=65535, initialBufferCapacity=65536]"); - } - }); - } - - @Test - public void testMaxRequestBufferSizeCannotBeLessThanInitialBufferSize() throws Exception { - assertMemoryLeak(() -> { - try (Sender ignored = Sender.builder(Sender.Transport.HTTP) - .address("localhost:1") - .maxBufferCapacity(100_000) - .bufferCapacity(200_000) - .build() - ) { - Assert.fail(); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "maximum buffer capacity cannot be less than initial buffer capacity [maximumBufferCapacity=100000, initialBufferCapacity=200000]"); - } - }); - } - - @Test - public void testMaxRetriesNotSupportedForTcp() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.TCP).address(LOCALHOST).retryTimeoutMillis(100).build(); - fail("max retries should not be supported for TCP"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "retrying is not supported for TCP protocol"); - } - }); - } - - @Test - public void testMinRequestThroughputCannotBeNegative() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.HTTP).address(LOCALHOST).minRequestThroughput(-100).build(); - fail("minimum request throughput must not be negative"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "minimum request throughput must not be negative [minRequestThroughput=-100]"); - } - }); - } - - @Test - public void testMinRequestThroughputNotSupportedForTcp() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.TCP).address(LOCALHOST).minRequestThroughput(1).build(); - fail("min request throughput is not be supported for TCP and the builder should fail-fast"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "minimum request throughput is not supported for TCP protocol"); - } - }); - } - - @Test - public void testPlainAuth_connectionRefused() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP) - .enableAuth("foo").authToken(AUTH_TOKEN_KEY1).address(LOCALHOST + ":19003"); - try { - builder.build(); - fail("connection refused should fail fast"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "could not connect"); - } - }); - } - - @Test - public void testPlainOldTokenNotSupportedForHttpProtocol() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.HTTP).address("localhost:9000").enableAuth("key").authToken(AUTH_TOKEN_KEY1).build(); - fail("HTTP token should not be supported for TCP"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "old token authentication is not supported for HTTP protocol"); - } - }); - } - - @Test - public void testPlain_connectionRefused() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).address(LOCALHOST + ":19003"); - try { - builder.build(); - fail("connection refused should fail fast"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "could not connect"); - } - }); - } - - @Test - public void testPortDoubleSet_firstAddressThenPort() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).address(LOCALHOST + ":9000"); - try { - builder.port(9000); - builder.build(); - fail("should not allow double port set"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "mismatch"); - } - }); - } - - @Test - public void testPortDoubleSet_firstPortThenAddress() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).port(9000); - try { - builder.address(LOCALHOST + ":9000"); - builder.build(); - fail("should not allow double port set"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "mismatch"); - } - }); - } - - @Test - public void testPortDoubleSet_firstPortThenPort() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).port(9000); - try { - builder.port(9000); - builder.build(); - fail("should not allow double port set"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "questdb server address not set"); - } - }); - } - - @Test - public void testSmallMaxNameLen() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.LineSenderBuilder ignored = Sender - .builder(Sender.Transport.TCP) - .maxNameLength(10); - fail("should not allow double buffer capacity set"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "max_name_len must be at least 16 bytes [max_name_len=10]"); - } - }); - } - - @Test - public void testTlsDoubleSet() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).enableTls(); - try { - builder.enableTls(); - fail("should not allow double tls set"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "already enabled"); - } - }); - } - - @Test - public void testTlsValidationDisabledButTlsNotEnabled() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP) - .advancedTls().disableCertificateValidation() - .address(LOCALHOST); - try { - builder.build(); - fail("should fail when TLS validation is disabled, but TLS not enabled"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "TLS was not enabled"); - } - }); - } - - @Test - public void testTlsValidationDisabledDoubleSet() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP) - .advancedTls().disableCertificateValidation(); - try { - builder.advancedTls().disableCertificateValidation(); - fail("should not allow double TLS validation disabled"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "TLS validation was already disabled"); - } - }); - } - - @Test - public void testTls_connectionRefused() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).enableTls().address(LOCALHOST + ":19003"); - try { - builder.build(); - fail("connection refused should fail fast"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "could not connect"); - } - }); - } - - @Test - public void testUsernamePasswordAuthNotSupportedForTcp() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.TCP).address(LOCALHOST).httpUsernamePassword("foo", "bar").build(); - fail("HTTP token should not be supported for TCP"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "username/password authentication is not supported for TCP protocol"); - } - }); - } - - private static void assertConfStrError(String conf, String expectedError) { - try { - try (Sender ignored = Sender.fromConfig(conf)) { - fail("should fail with bad conf string"); - } - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), expectedError); - } - } - - private static void assertConfStrOk(String... params) { - StringBuilder sb = new StringBuilder(); - sb.append("http").append("::"); - shuffle(params); - for (int i = 0; i < params.length; i++) { - sb.append(params[i]).append(";"); - } - assertConfStrOk(sb.toString()); - } - - private static void assertConfStrOk(String conf) { - Sender.fromConfig(conf).close(); - } - - private static void shuffle(String[] input) { - for (int i = 0; i < input.length; i++) { - int j = (int) (Math.random() * input.length); - String tmp = input[i]; - input[i] = input[j]; - input[j] = tmp; - } - } -} diff --git a/core/src/test/java/io/questdb/client/test/cairo/ColumnTypeTest.java b/core/src/test/java/io/questdb/client/test/cairo/ColumnTypeTest.java index 0453bdc..9d40c3a 100644 --- a/core/src/test/java/io/questdb/client/test/cairo/ColumnTypeTest.java +++ b/core/src/test/java/io/questdb/client/test/cairo/ColumnTypeTest.java @@ -30,14 +30,8 @@ public class ColumnTypeTest { @Test - public void testArrayWithWeakDims() { - int arrayType = ColumnType.encodeArrayTypeWithWeakDims(ColumnType.DOUBLE, true); - Assert.assertTrue(ColumnType.isArray(arrayType)); - // arrays with weak dimensions are considered undefined - Assert.assertEquals(ColumnType.DOUBLE, ColumnType.decodeArrayElementType(arrayType)); - Assert.assertEquals(-1, ColumnType.decodeWeakArrayDimensionality(arrayType)); - - arrayType = ColumnType.encodeArrayType(ColumnType.DOUBLE, 5); + public void testArrayEncoding() { + int arrayType = ColumnType.encodeArrayType(ColumnType.DOUBLE, 5); Assert.assertTrue(ColumnType.isArray(arrayType)); Assert.assertEquals(ColumnType.DOUBLE, ColumnType.decodeArrayElementType(arrayType)); Assert.assertEquals(5, ColumnType.decodeWeakArrayDimensionality(arrayType)); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/http/HttpHeaderParserTest.java b/core/src/test/java/io/questdb/client/test/cutlass/http/HttpHeaderParserTest.java index 78a9a08..6974d0b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/http/HttpHeaderParserTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/http/HttpHeaderParserTest.java @@ -33,6 +33,7 @@ import io.questdb.client.std.str.DirectUtf8String; import io.questdb.client.std.str.Utf8String; import io.questdb.client.test.tools.TestUtils; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Assert; import org.junit.Test; @@ -53,7 +54,7 @@ public class HttpHeaderParserTest { @Test public void testContentLengthLarge() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { String v = "Content-Length: 81136060058\r\n" + "\r\n"; long p = TestUtils.toMemory(v); @@ -135,7 +136,7 @@ public void testProtocolLineFuzz() { @Test public void testQueryDanglingEncoding() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { String v = "GET /status?x=1&a=% HTTP/1.1\r\n" + "\r\n"; long p = TestUtils.toMemory(v); @@ -152,7 +153,7 @@ public void testQueryDanglingEncoding() throws Exception { @Test public void testQueryInvalidEncoding() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { String v = "GET /status?x=1&a=%i6b&c&d=x HTTP/1.1\r\n" + "\r\n"; long p = TestUtils.toMemory(v); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/http/TestHttpClient.java b/core/src/test/java/io/questdb/client/test/cutlass/http/TestHttpClient.java index e55f625..c9aa65a 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/http/TestHttpClient.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/http/TestHttpClient.java @@ -32,7 +32,6 @@ import io.questdb.client.std.str.MutableUtf8Sink; import io.questdb.client.std.str.Utf8Sequence; import io.questdb.client.std.str.Utf8StringSink; -import io.questdb.client.std.str.Utf8s; import io.questdb.client.test.tools.TestUtils; import org.jetbrains.annotations.Nullable; import org.junit.Assert; @@ -489,7 +488,8 @@ protected String reqToSink0( @SuppressWarnings("resource") HttpClient.ResponseHeaders rsp = req.send(); rsp.await(); - String statusCode = Utf8s.toString(rsp.getStatusCode()); + Utf8Sequence sc = rsp.getStatusCode(); + String statusCode = sc == null ? null : sc.toString(); sink.clear(); rsp.getResponse().copyTextTo(sink); return statusCode; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java new file mode 100644 index 0000000..ba7c26d --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java @@ -0,0 +1,282 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.http.client; + +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.cutlass.http.client.HttpClientException; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; +import io.questdb.client.cutlass.http.client.WebSocketSendBuffer; +import io.questdb.client.network.NetworkFacade; +import io.questdb.client.network.PlainSocketFactory; +import io.questdb.client.network.Socket; +import io.questdb.client.network.TlsSessionInitFailedException; +import io.questdb.client.std.Unsafe; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Field; + +public class WebSocketClientTest { + + @Test + public void testSendCloseFrameDoesNotClobberSendBuffer() throws Exception { + assertMemoryLeak(() -> { + try (StubWebSocketClient client = new StubWebSocketClient()) { + WebSocketSendBuffer sendBuffer = client.getSendBuffer(); + + // User starts building a data frame + sendBuffer.beginFrame(); + sendBuffer.putLong(0xDEADBEEFL); + int posBeforeClose = sendBuffer.getWritePos(); + Assert.assertTrue("sendBuffer should have data", posBeforeClose > 0); + + // sendCloseFrame() should use controlFrameBuffer, not sendBuffer + try { + client.sendCloseFrame(1000, null, 1000); + } catch (HttpClientException ignored) { + // Expected: doSend() fails because there's no real socket + } + + // Verify sendBuffer was NOT clobbered + Assert.assertEquals( + "sendCloseFrame() must not reset the main sendBuffer", + posBeforeClose, + sendBuffer.getWritePos() + ); + } + }); + } + + @Test + public void testSendPingDoesNotClobberSendBuffer() throws Exception { + assertMemoryLeak(() -> { + try (StubWebSocketClient client = new StubWebSocketClient()) { + // Set upgraded=true so checkConnected() passes + setField(client, "upgraded", true); + + WebSocketSendBuffer sendBuffer = client.getSendBuffer(); + + // User starts building a data frame + sendBuffer.beginFrame(); + sendBuffer.putLong(0xCAFEBABEL); + int posBeforePing = sendBuffer.getWritePos(); + Assert.assertTrue("sendBuffer should have data", posBeforePing > 0); + + // sendPing() should use controlFrameBuffer, not sendBuffer + try { + client.sendPing(1000); + } catch (HttpClientException ignored) { + // Expected: doSend() fails because there's no real socket + } + + // Verify sendBuffer was NOT clobbered + Assert.assertEquals( + "sendPing() must not reset the main sendBuffer", + posBeforePing, + sendBuffer.getWritePos() + ); + } + }); + } + + @Test + public void testRecvOrTimeoutPropagatesNonTimeoutError() throws Exception { + assertMemoryLeak(() -> { + try (RecvTestWebSocketClient client = new RecvTestWebSocketClient()) { + setField(client, "upgraded", true); + + // socket.recv() returns 0, triggering the ioWait path + // ioWait throws a non-timeout error (e.g., queue/poll failure) + client.ioWaitAction = () -> { + throw new HttpClientException("queue error [errno=").put(5).put(']'); + }; + + WebSocketFrameHandler noOpHandler = new WebSocketFrameHandler() { + @Override + public void onBinaryMessage(long payloadPtr, int payloadLen) { + } + + @Override + public void onClose(int code, String reason) { + } + }; + + try { + client.receiveFrame(noOpHandler, 1000); + Assert.fail("expected HttpClientException for queue error"); + } catch (HttpClientException e) { + Assert.assertFalse("non-timeout error must not be flagged as timeout", e.isTimeout()); + Assert.assertTrue( + "expected queue error message, got: " + e.getMessage(), + e.getMessage().contains("queue error") + ); + } + } + }); + } + + @Test + public void testRecvOrTimeoutReturnsFalseOnTimeout() throws Exception { + assertMemoryLeak(() -> { + try (RecvTestWebSocketClient client = new RecvTestWebSocketClient()) { + setField(client, "upgraded", true); + + // socket.recv() returns 0, triggering the ioWait path + // ioWait throws a timeout error + client.ioWaitAction = () -> { + throw new HttpClientException("timed out [errno=").put(0).put(']').flagAsTimeout(); + }; + + WebSocketFrameHandler noOpHandler = new WebSocketFrameHandler() { + @Override + public void onBinaryMessage(long payloadPtr, int payloadLen) { + } + + @Override + public void onClose(int code, String reason) { + } + }; + + boolean result = client.receiveFrame(noOpHandler, 1000); + Assert.assertFalse("receiveFrame should return false on timeout", result); + } + }); + } + + private static void setField(Object obj, String fieldName, Object value) throws Exception { + Class clazz = obj.getClass(); + while (clazz != null) { + try { + Field field = clazz.getDeclaredField(fieldName); + field.setAccessible(true); + field.set(obj, value); + return; + } catch (NoSuchFieldException e) { + clazz = clazz.getSuperclass(); + } + } + throw new NoSuchFieldException(fieldName); + } + + /** + * Minimal concrete WebSocketClient that throws on any I/O, + * allowing us to test buffer management without a real socket. + */ + private static class StubWebSocketClient extends WebSocketClient { + + StubWebSocketClient() { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + } + + @Override + protected void ioWait(int timeout, int op) { + throw new HttpClientException("stub: no socket"); + } + + @Override + protected void setupIoWait() { + // no-op + } + } + + /** + * WebSocketClient subclass with a fake socket that always returns 0 + * from recv(), forcing the ioWait path in recvOrTimeout(). + */ + private static class RecvTestWebSocketClient extends WebSocketClient { + Runnable ioWaitAction; + + RecvTestWebSocketClient() { + super(DefaultHttpClientConfiguration.INSTANCE, (nf, log) -> new FakeSocket()); + } + + @Override + protected void ioWait(int timeout, int op) { + ioWaitAction.run(); + } + + @Override + protected void setupIoWait() { + // no-op + } + } + + /** + * Minimal Socket that always returns 0 from recv() (no data available), + * triggering the ioWait path in recvOrTimeout(). + */ + private static class FakeSocket implements Socket { + + @Override + public void close() { + } + + @Override + public int getFd() { + return 0; + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public void of(int fd) { + } + + @Override + public int recv(long bufferPtr, int bufferLen) { + return 0; + } + + @Override + public int send(long bufferPtr, int bufferLen) { + return 0; + } + + @Override + public void startTlsSession(CharSequence peerName) throws TlsSessionInitFailedException { + throw new UnsupportedOperationException(); + } + + @Override + public boolean supportsTls() { + return false; + } + + @Override + public int tlsIO(int readinessFlags) { + return 0; + } + + @Override + public boolean wantsTlsWrite() { + return false; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cairo/arr/BorrowedArray.java b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketSendBufferTest.java similarity index 51% rename from core/src/main/java/io/questdb/client/cairo/arr/BorrowedArray.java rename to core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketSendBufferTest.java index 4a7809b..dde2ccf 100644 --- a/core/src/main/java/io/questdb/client/cairo/arr/BorrowedArray.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketSendBufferTest.java @@ -22,26 +22,28 @@ * ******************************************************************************/ -package io.questdb.client.cairo.arr; +package io.questdb.client.test.cutlass.http.client; -import io.questdb.client.cairo.ColumnType; -import io.questdb.client.std.Mutable; +import io.questdb.client.cutlass.http.client.WebSocketSendBuffer; +import io.questdb.client.std.Unsafe; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; +import org.junit.Test; -public class BorrowedArray extends MutableArray implements Mutable { +import static org.junit.Assert.assertEquals; - public BorrowedArray() { - this.flatView = new BorrowedFlatArrayView(); - } +public class WebSocketSendBufferTest { - /** - * Resets to an invalid array. - */ - @Override - public void clear() { - this.type = ColumnType.UNDEFINED; - borrowedFlatView().reset(); - shape.clear(); - strides.clear(); + @Test + public void testPutUtf8InvalidSurrogatePair() throws Exception { + assertMemoryLeak(() -> { + try (WebSocketSendBuffer buf = new WebSocketSendBuffer(256)) { + // High surrogate \uD800 followed by non-low-surrogate 'X'. + // Should produce '?' for the lone high surrogate, then 'X'. + buf.putUtf8("\uD800X"); + assertEquals(2, buf.getWritePos()); + assertEquals((byte) '?', Unsafe.getUnsafe().getByte(buf.getBufferPtr())); + assertEquals((byte) 'X', Unsafe.getUnsafe().getByte(buf.getBufferPtr() + 1)); + } + }); } - -} \ No newline at end of file +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/json/JsonLexerTest.java b/core/src/test/java/io/questdb/client/test/cutlass/json/JsonLexerTest.java index 4738d39..715c5e8 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/json/JsonLexerTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/json/JsonLexerTest.java @@ -33,6 +33,7 @@ import io.questdb.client.std.Mutable; import io.questdb.client.std.Unsafe; import io.questdb.client.test.tools.TestUtils; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; @@ -78,7 +79,7 @@ public void testBreakOnValue() throws Exception { @Test public void testCacheDisabled() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { String json = "{\"a\":1, \"b\": \"123456789012345678901234567890\"}"; int len = json.length(); long address = TestUtils.toMemory(json); @@ -251,7 +252,7 @@ public void testSimpleJson() throws Exception { @Test public void testStringTooLong() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { String json = "{\"a\":1, \"b\": \"123456789012345678901234567890\"]}"; int len = json.length() - 6; long address = TestUtils.toMemory(json); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/AbstractLineSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/line/AbstractLineSenderTest.java deleted file mode 100644 index 9180b21..0000000 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/AbstractLineSenderTest.java +++ /dev/null @@ -1,135 +0,0 @@ -package io.questdb.client.test.cutlass.line; - -import io.questdb.client.cutlass.line.AbstractLineSender; -import io.questdb.client.test.AbstractQdbTest; -import org.junit.Assume; -import org.junit.BeforeClass; - -import java.lang.reflect.Array; - -public class AbstractLineSenderTest extends AbstractQdbTest { - private static final int DEFAULT_ILP_TCP_PORT = 9009; - private static final int DEFAULT_ILP_UDP_PORT = 9009; - - public static T createDoubleArray(int... shape) { - int[] indices = new int[shape.length]; - return buildNestedArray(ArrayDataType.DOUBLE, shape, 0, indices); - } - - @BeforeClass - public static void setUpStatic() { - AbstractQdbTest.setUpStatic(); - Assume.assumeTrue(getQuestDBRunning()); - } - - @SuppressWarnings("unchecked") - private static T buildNestedArray(ArrayDataType dataType, int[] shape, int currentDim, int[] indices) { - if (currentDim == shape.length - 1) { - Object arr = dataType.createArray(shape[currentDim]); - for (int i = 0; i < Array.getLength(arr); i++) { - indices[currentDim] = i; - dataType.setElement(arr, i, indices); - } - return (T) arr; - } else { - Class componentType = dataType.getComponentType(shape.length - currentDim - 1); - Object arr = Array.newInstance(componentType, shape[currentDim]); - for (int i = 0; i < shape[currentDim]; i++) { - indices[currentDim] = i; - Object subArr = buildNestedArray(dataType, shape, currentDim + 1, indices); - Array.set(arr, i, subArr); - } - return (T) arr; - } - } - - /** - * Get ILP TCP port. - */ - protected static int getIlpTcpPort() { - return getConfigInt("QUESTDB_ILP_TCP_PORT", "questdb.ilp.tcp.port", DEFAULT_ILP_TCP_PORT); - } - - /** - * Get ILP UDP port. - */ - protected static int getIlpUdpPort() { - return getConfigInt("QUESTDB_ILP_UDP_PORT", "questdb.ilp.udp.port", DEFAULT_ILP_UDP_PORT); - } - - /** - * Send data using the sender and assert the expected row count. - * This method flushes the sender, waits for UDP to settle, and polls for the expected row count. - *

    - * UDP is fire-and-forget, so we need extra delay to ensure the server has processed the data. - * - * @param sender the sender to flush - * @param tableName the table to check - * @param expectedRowCount the expected number of rows - */ - protected void flushAndAssertRowCount(AbstractLineSender sender, String tableName, int expectedRowCount) throws Exception { - sender.flush(); - assertTableSizeEventually(tableName, expectedRowCount); - } - - private enum ArrayDataType { - DOUBLE(double.class) { - @Override - public Object createArray(int length) { - return new double[length]; - } - - @Override - public void setElement(Object array, int index, int[] indices) { - double[] arr = (double[]) array; - double product = 1.0; - for (int idx : indices) { - product *= (idx + 1); - } - arr[index] = product; - } - }, - LONG(long.class) { - @Override - public Object createArray(int length) { - return new long[length]; - } - - @Override - public void setElement(Object array, int index, int[] indices) { - long[] arr = (long[]) array; - long product = 1L; - for (int idx : indices) { - product *= (idx + 1); - } - arr[index] = product; - } - }; - - private final Class baseType; - private final Class[] componentTypes = new Class[17]; // 支持最多16维 - - ArrayDataType(Class baseType) { - this.baseType = baseType; - initComponentTypes(); - } - - public abstract Object createArray(int length); - - public Class getComponentType(int dimsRemaining) { - if (dimsRemaining < 0 || dimsRemaining > 16) { - throw new RuntimeException("Array dimension too large"); - } - return componentTypes[dimsRemaining]; - } - - public abstract void setElement(Object array, int index, int[] indices); - - private void initComponentTypes() { - componentTypes[0] = baseType; - for (int dim = 1; dim <= 16; dim++) { - componentTypes[dim] = Array.newInstance(componentTypes[dim - 1], 0).getClass(); - } - } - } -} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java new file mode 100644 index 0000000..2711592 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java @@ -0,0 +1,492 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.line; + +import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Test; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; +import static org.junit.Assert.fail; + +/** + * Unit tests for LineSenderBuilder that don't require a running QuestDB instance. + * Tests that require an actual QuestDB connection have been moved to integration tests. + */ +public class LineSenderBuilderTest { + private static final String AUTH_TOKEN_KEY1 = "UvuVb1USHGRRT08gEnwN2zGZrvM4MsLQ5brgF6SVkAw="; + private static final String LOCALHOST = "localhost"; + private static final char[] TRUSTSTORE_PASSWORD = "questdb".toCharArray(); + private static final String TRUSTSTORE_PATH = "/keystore/server.keystore"; + + @Test + public void testAddressDoubleSet_firstAddressThenAddress() throws Exception { + assertMemoryLeak(() -> assertThrows("mismatch", + Sender.builder(Sender.Transport.TCP).address(LOCALHOST).address("127.0.0.1"))); + } + + @Test + public void testAddressEmpty() throws Exception { + assertMemoryLeak(() -> assertThrows("address cannot be empty", + () -> Sender.builder(Sender.Transport.TCP).address(""))); + } + + @Test + public void testAddressEndsWithColon() throws Exception { + assertMemoryLeak(() -> assertThrows("invalid address", + () -> Sender.builder(Sender.Transport.TCP).address("foo:"))); + } + + @Test + public void testAddressNull() throws Exception { + assertMemoryLeak(() -> assertThrows("null", + () -> Sender.builder(Sender.Transport.TCP).address(null))); + } + + @Test + public void testAuthDoubleSet() throws Exception { + assertMemoryLeak(() -> assertThrows("already configured", + () -> Sender.builder(Sender.Transport.TCP).enableAuth("foo").authToken(AUTH_TOKEN_KEY1).enableAuth("bar"))); + } + + @Test + public void testAuthTooSmallBuffer() throws Exception { + assertMemoryLeak(() -> assertThrows("minimalCapacity", + Sender.builder(Sender.Transport.TCP) + .enableAuth("foo").authToken(AUTH_TOKEN_KEY1).address(LOCALHOST + ":9001") + .bufferCapacity(1))); + } + + @Test + public void testAuthWithBadToken() throws Exception { + assertMemoryLeak(() -> assertThrows("could not import token", + () -> Sender.builder(Sender.Transport.TCP).enableAuth("foo").authToken("bar token"))); + } + + @Test + public void testAutoFlushIntervalMustBePositive() { + assertThrows("auto flush interval cannot be negative [autoFlushIntervalMillis=0]", + () -> Sender.builder(Sender.Transport.HTTP).autoFlushIntervalMillis(0)); + assertThrows("auto flush interval cannot be negative [autoFlushIntervalMillis=-1]", + () -> Sender.builder(Sender.Transport.HTTP).autoFlushIntervalMillis(-1)); + } + + @Test + public void testAutoFlushIntervalNotSupportedForTcp() throws Exception { + assertMemoryLeak(() -> assertThrows("auto flush interval is not supported for TCP protocol", + Sender.builder(Sender.Transport.TCP).address(LOCALHOST).autoFlushIntervalMillis(1))); + } + + @Test + public void testAutoFlushInterval_afterAutoFlushDisabled() throws Exception { + assertMemoryLeak(() -> assertThrows("cannot set auto flush interval when interval based auto-flush is already disabled", + () -> Sender.builder(Sender.Transport.HTTP).disableAutoFlush().autoFlushIntervalMillis(1))); + } + + @Test + public void testAutoFlushInterval_doubleConfiguration() throws Exception { + assertMemoryLeak(() -> assertThrows("auto flush interval was already configured [autoFlushIntervalMillis=1]", + () -> Sender.builder(Sender.Transport.HTTP).autoFlushIntervalMillis(1).autoFlushIntervalMillis(1))); + } + + @Test + public void testAutoFlushRowsCannotBeNegative() { + assertThrows("auto flush rows cannot be negative [autoFlushRows=-1]", + () -> Sender.builder(Sender.Transport.HTTP).autoFlushRows(-1)); + } + + @Test + public void testAutoFlushRowsNotSupportedForTcp() throws Exception { + assertMemoryLeak(() -> assertThrows("auto flush rows is not supported for TCP protocol", + Sender.builder(Sender.Transport.TCP).address(LOCALHOST).autoFlushRows(1))); + } + + @Test + public void testAutoFlushRows_doubleConfiguration() throws Exception { + assertMemoryLeak(() -> assertThrows("auto flush rows was already configured [autoFlushRows=1]", + () -> Sender.builder(Sender.Transport.HTTP).autoFlushRows(1).autoFlushRows(1))); + } + + @Test + public void testBufferSizeDoubleSet() throws Exception { + assertMemoryLeak(() -> assertThrows("already configured", + () -> Sender.builder(Sender.Transport.TCP).bufferCapacity(1024).bufferCapacity(1024))); + } + + @Test + public void testConfStringValidation() throws Exception { + assertMemoryLeak(() -> { + assertConfStrError("foo", "invalid schema [schema=foo, supported-schemas=[http, https, tcp, tcps, ws, wss, udp]]"); + assertConfStrError("badschema::addr=bar;", "invalid schema [schema=badschema, supported-schemas=[http, https, tcp, tcps, ws, wss, udp]]"); + assertConfStrError("http::addr=localhost:-1;", "invalid port [port=-1]"); + assertConfStrError("http::auto_flush=on;", "addr is missing"); + assertConfStrError("http::addr=localhost;tls_roots=/some/path;", "tls_roots was configured, but tls_roots_password is missing"); + assertConfStrError("http::addr=localhost;tls_roots_password=hunter123;", "tls_roots_password was configured, but tls_roots is missing"); + assertConfStrError("tcp::addr=localhost;user=foo;", "token cannot be empty nor null"); + assertConfStrError("tcp::addr=localhost;username=foo;", "token cannot be empty nor null"); + assertConfStrError("tcp::addr=localhost;token=foo;", "TCP token is configured, but user is missing"); + assertConfStrError("http::addr=localhost;user=foo;", "password cannot be empty nor null"); + assertConfStrError("http::addr=localhost;username=foo;", "password cannot be empty nor null"); + assertConfStrError("http::addr=localhost;pass=foo;", "password is configured, but username is missing"); + assertConfStrError("http::addr=localhost;password=foo;", "password is configured, but username is missing"); + assertConfStrError("tcp::addr=localhost;pass=foo;", "password is not supported for TCP protocol"); + assertConfStrError("tcp::addr=localhost;password=foo;", "password is not supported for TCP protocol"); + assertConfStrError("tcp::addr=localhost;retry_timeout=;", "retry_timeout cannot be empty"); + assertConfStrError("tcp::addr=localhost;max_buf_size=;", "max_buf_size cannot be empty"); + assertConfStrError("tcp::addr=localhost;init_buf_size=;", "init_buf_size cannot be empty"); + assertConfStrError("http::addr=localhost:8080;tls_verify=unsafe_off;", "TLS validation disabled, but TLS was not enabled"); + assertConfStrError("http::addr=localhost:8080;tls_verify=bad;", "invalid tls_verify [value=bad, allowed-values=[on, unsafe_off]]"); + assertConfStrError("tcps::addr=localhost;pass=unsafe_off;", "password is not supported for TCP protocol"); + assertConfStrError("tcps::addr=localhost;password=unsafe_off;", "password is not supported for TCP protocol"); + assertConfStrError("http::addr=localhost:8080;max_buf_size=-32;", "maximum buffer capacity cannot be less than initial buffer capacity [maximumBufferCapacity=-32, initialBufferCapacity=65536]"); + assertConfStrError("http::addr=localhost:8080;max_buf_size=notanumber;", "invalid max_buf_size [value=notanumber]"); + assertConfStrError("http::addr=localhost:8080;init_buf_size=notanumber;", "invalid init_buf_size [value=notanumber]"); + assertConfStrError("http::addr=localhost:8080;init_buf_size=-42;", "buffer capacity cannot be negative [capacity=-42]"); + assertConfStrError("http::addr=localhost:8080;auto_flush_rows=0;", "invalid auto_flush_rows [value=0]"); + assertConfStrError("http::addr=localhost:8080;auto_flush_rows=notanumber;", "invalid auto_flush_rows [value=notanumber]"); + assertConfStrError("http::addr=localhost:8080;auto_flush=invalid;", "invalid auto_flush [value=invalid, allowed-values=[on, off]]"); + assertConfStrError("http::addr=localhost:8080;auto_flush=off;auto_flush_rows=100;", "cannot set auto flush rows when auto-flush is already disabled"); + assertConfStrError("http::addr=localhost:8080;auto_flush_rows=100;auto_flush=off;", "auto flush rows was already configured [autoFlushRows=100]"); + assertConfStrError("HTTP::addr=localhost;", "invalid schema [schema=HTTP, supported-schemas=[http, https, tcp, tcps, ws, wss, udp]]"); + assertConfStrError("HTTPS::addr=localhost;", "invalid schema [schema=HTTPS, supported-schemas=[http, https, tcp, tcps, ws, wss, udp]]"); + assertConfStrError("TCP::addr=localhost;", "invalid schema [schema=TCP, supported-schemas=[http, https, tcp, tcps, ws, wss, udp]]"); + assertConfStrError("TCPS::addr=localhost;", "invalid schema [schema=TCPS, supported-schemas=[http, https, tcp, tcps, ws, wss, udp]]"); + assertConfStrError("http::addr=localhost;auto_flush=off;auto_flush_interval=1;", "cannot set auto flush interval when interval based auto-flush is already disabled"); + assertConfStrError("http::addr=localhost;auto_flush=off;auto_flush_rows=1;", "cannot set auto flush rows when auto-flush is already disabled"); + assertConfStrError("http::addr=localhost;auto_flush_bytes=1024;", "auto_flush_bytes is only supported for TCP transport"); + assertConfStrError("http::addr=localhost;protocol_version=10", "current client only supports protocol version 1(text format for all datatypes), 2(binary format for part datatypes), 3(decimal datatype) or explicitly unset"); + assertConfStrError("http::addr=localhost:48884;max_name_len=10;", "max_name_len must be at least 16 bytes [max_name_len=10]"); + assertConfStrError("ws::addr=localhost;token=foo;", "token is not supported for WebSocket protocol"); + assertConfStrError("wss::addr=localhost;token=foo;", "token is not supported for WebSocket protocol"); + + assertConfStrOk("addr=localhost:8080", "auto_flush_rows=100", "protocol_version=1"); + assertConfStrOk("addr=localhost:8080", "auto_flush=on", "auto_flush_rows=100", "protocol_version=2"); + assertConfStrOk("addr=localhost:8080", "auto_flush_rows=100", "auto_flush=on", "protocol_version=2"); + assertConfStrOk("addr=localhost", "auto_flush=on", "protocol_version=2"); + assertConfStrOk("addr=localhost:8080", "max_name_len=1024", "protocol_version=2"); + + assertConfStrError("tcp::addr=localhost;auto_flush_bytes=1024;init_buf_size=2048;", "TCP transport requires init_buf_size and auto_flush_bytes to be set to the same value [init_buf_size=2048, auto_flush_bytes=1024]"); + assertConfStrError("tcp::addr=localhost;init_buf_size=1024;auto_flush_bytes=2048;", "TCP transport requires init_buf_size and auto_flush_bytes to be set to the same value [init_buf_size=1024, auto_flush_bytes=2048]"); + assertConfStrError("tcp::addr=localhost;auto_flush_bytes=off;", "TCP transport must have auto_flush_bytes enabled"); + + assertConfStrOk("http::addr=localhost;auto_flush=off;protocol_version=2;"); + assertConfStrOk("http::addr=localhost;protocol_version=2;"); + assertConfStrOk("http::addr=localhost;auto_flush_interval=off;protocol_version=2;"); + assertConfStrOk("http::addr=localhost;auto_flush_rows=off;protocol_version=2;"); + assertConfStrOk("http::addr=localhost;auto_flush_interval=off;auto_flush_rows=off;protocol_version=1;"); + assertConfStrOk("http::addr=localhost;auto_flush_interval=off;auto_flush_rows=1;protocol_version=2;"); + assertConfStrOk("http::addr=localhost;auto_flush_rows=off;auto_flush_interval=1;protocol_version=2;"); + assertConfStrOk("http::addr=localhost;auto_flush_interval=off;auto_flush_rows=off;auto_flush=off;protocol_version=2;"); + assertConfStrOk("http::addr=localhost;auto_flush=off;auto_flush_interval=off;auto_flush_rows=off;protocol_version=1;"); + assertConfStrOk("http::addr=localhost:8080;protocol_version=2;"); + assertConfStrOk("http::addr=localhost:8080;token=foo;protocol_version=2;"); + assertConfStrOk("http::addr=localhost:8080;token=foo=bar;protocol_version=2;"); + assertConfStrOk("addr=localhost:8080", "token=foo", "retry_timeout=1000", "max_buf_size=1000000", "protocol_version=2"); + assertConfStrOk("addr=localhost:8080", "token=foo", "retry_timeout=1000", "max_buf_size=1000000", "protocol_version=1"); + assertConfStrOk("http::addr=localhost:8080;token=foo;max_buf_size=1000000;retry_timeout=1000;protocol_version=2;"); + assertConfStrOk("https::addr=localhost:8080;tls_verify=unsafe_off;auto_flush_rows=100;protocol_version=2;"); + assertConfStrOk("https::addr=localhost:8080;tls_verify=unsafe_off;auto_flush_rows=100;protocol_version=2;max_name_len=256;"); + assertConfStrOk("https::addr=localhost:8080;tls_verify=on;protocol_version=2;"); + assertConfStrOk("https::addr=localhost:8080;tls_verify=on;protocol_version=3;"); + assertConfStrError("https::addr=2001:0db8:85a3:0000:0000:8a2e:0370:7334;tls_verify=on;", "cannot parse a port from the address, use IPv4 address or a domain name [address=2001:0db8:85a3:0000:0000:8a2e:0370:7334]"); + assertConfStrError("https::addr=[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:9000;tls_verify=on;", "cannot parse a port from the address, use IPv4 address or a domain name [address=[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:9000]"); + }); + } + + @Test + public void testCustomTruststoreButTlsNotEnabled() throws Exception { + assertMemoryLeak(() -> assertThrows("TLS was not enabled", + Sender.builder(Sender.Transport.TCP) + .advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD) + .address(LOCALHOST))); + } + + @Test + public void testCustomTruststoreDoubleSet() throws Exception { + assertMemoryLeak(() -> assertThrows("already configured", + () -> Sender.builder(Sender.Transport.TCP) + .advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD) + .advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD))); + } + + @Test + public void testCustomTruststorePasswordCannotBeNull() { + assertThrows("trust store password cannot be null", + () -> Sender.builder(Sender.Transport.TCP).advancedTls().customTrustStore(TRUSTSTORE_PATH, null)); + } + + @Test + public void testCustomTruststorePathCannotBeBlank() { + assertThrows("trust store path cannot be empty nor null", + () -> Sender.builder(Sender.Transport.TCP).advancedTls().customTrustStore("", TRUSTSTORE_PASSWORD)); + assertThrows("trust store path cannot be empty nor null", + () -> Sender.builder(Sender.Transport.TCP).advancedTls().customTrustStore(null, TRUSTSTORE_PASSWORD)); + } + + @Test + public void testDisableAutoFlushNotSupportedForTcp() throws Exception { + assertMemoryLeak(() -> assertThrows("auto-flush is not supported for TCP protocol", + Sender.builder(Sender.Transport.TCP).address(LOCALHOST).disableAutoFlush())); + } + + @Test + public void testDnsResolutionFail() throws Exception { + assertMemoryLeak(() -> assertThrows("could not resolve", + Sender.builder(Sender.Transport.TCP).address("this-domain-does-not-exist-i-hope-better-to-use-a-silly-tld.silly-tld"))); + } + + @Test + public void testDuplicatedAddresses() throws Exception { + assertMemoryLeak(() -> { + assertThrows("duplicated addresses are not allowed [address=localhost:9000]", + () -> Sender.builder(Sender.Transport.TCP).address("localhost:9000").address("localhost:9000")); + assertThrows("duplicated addresses are not allowed [address=localhost:9000]", + () -> Sender.builder(Sender.Transport.TCP).address("localhost:9000").address("localhost:9001").address("localhost:9000")); + }); + } + + @Test + public void testDuplicatedAddressesWithDifferentPortsAllowed() throws Exception { + assertMemoryLeak(() -> Sender.builder(Sender.Transport.TCP).address("localhost:9000").address("localhost:9001")); + } + + @Test + public void testDuplicatedAddressesWithNoPortsAllowed() throws Exception { + assertMemoryLeak(() -> { + Sender.builder(Sender.Transport.TCP).address("localhost:9000").address("localhost"); + Sender.builder(Sender.Transport.TCP).address("localhost").address("localhost:9000"); + }); + } + + @Test + public void testFailFastWhenSetCustomTrustStoreTwice() { + assertThrows("already configured", + () -> Sender.builder(Sender.Transport.TCP) + .advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD) + .advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD)); + } + + @Test + public void testFirstTlsValidationDisabledThenCustomTruststore() throws Exception { + assertMemoryLeak(() -> assertThrows("TLS validation was already disabled", + () -> Sender.builder(Sender.Transport.TCP) + .advancedTls().disableCertificateValidation() + .advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD))); + } + + @Test + public void testHostNorAddressSet() throws Exception { + assertMemoryLeak(() -> assertThrows("server address not set", + Sender.builder(Sender.Transport.TCP))); + } + + @Test + public void testHttpTokenNotSupportedForTcp() throws Exception { + assertMemoryLeak(() -> assertThrows("HTTP token authentication is not supported for TCP protocol", + Sender.builder(Sender.Transport.TCP).address(LOCALHOST).httpToken("foo"))); + } + + @Test + public void testInvalidHttpTimeout() throws Exception { + assertMemoryLeak(() -> { + assertThrows("HTTP timeout must be positive [timeout=0]", + () -> Sender.builder(Sender.Transport.HTTP).address("someurl").httpTimeoutMillis(0)); + assertThrows("HTTP timeout must be positive [timeout=-1]", + () -> Sender.builder(Sender.Transport.HTTP).address("someurl").httpTimeoutMillis(-1)); + assertThrows("HTTP timeout was already configured [timeout=100]", + () -> Sender.builder(Sender.Transport.HTTP).address("someurl").httpTimeoutMillis(100).httpTimeoutMillis(200)); + assertThrows("HTTP timeout is not supported for TCP protocol", + Sender.builder(Sender.Transport.TCP).address("localhost").httpTimeoutMillis(5000)); + }); + } + + @Test + public void testInvalidRetryTimeout() { + assertThrows("retry timeout cannot be negative [retryTimeoutMillis=-1]", + () -> Sender.builder(Sender.Transport.HTTP).retryTimeoutMillis(-1)); + assertThrows("retry timeout was already configured [retryTimeoutMillis=100]", + () -> Sender.builder(Sender.Transport.HTTP).retryTimeoutMillis(100).retryTimeoutMillis(200)); + } + + @Test + public void testMalformedPortInAddress() throws Exception { + assertMemoryLeak(() -> assertThrows("cannot parse a port from the address", + () -> Sender.builder(Sender.Transport.TCP).address("foo:nonsense12334"))); + } + + @Test + public void testMaxRequestBufferSizeCannotBeLessThanDefault() throws Exception { + assertMemoryLeak(() -> assertThrows("maximum buffer capacity cannot be less than initial buffer capacity [maximumBufferCapacity=65535, initialBufferCapacity=65536]", + () -> Sender.builder(Sender.Transport.HTTP).address("localhost:1").maxBufferCapacity(65535))); + } + + @Test + public void testMaxRequestBufferSizeCannotBeLessThanInitialBufferSize() throws Exception { + assertMemoryLeak(() -> assertThrows("maximum buffer capacity cannot be less than initial buffer capacity [maximumBufferCapacity=100000, initialBufferCapacity=200000]", + Sender.builder(Sender.Transport.HTTP).address("localhost:1").maxBufferCapacity(100_000).bufferCapacity(200_000))); + } + + @Test + public void testMaxRetriesNotSupportedForTcp() throws Exception { + assertMemoryLeak(() -> assertThrows("retrying is not supported for TCP protocol", + () -> Sender.builder(Sender.Transport.TCP).address(LOCALHOST).retryTimeoutMillis(100))); + } + + @Test + public void testMinRequestThroughputCannotBeNegative() throws Exception { + assertMemoryLeak(() -> assertThrows("minimum request throughput must not be negative [minRequestThroughput=-100]", + () -> Sender.builder(Sender.Transport.HTTP).address(LOCALHOST).minRequestThroughput(-100))); + } + + @Test + public void testMinRequestThroughputNotSupportedForTcp() throws Exception { + assertMemoryLeak(() -> assertThrows("minimum request throughput is not supported for TCP protocol", + Sender.builder(Sender.Transport.TCP).address(LOCALHOST).minRequestThroughput(1))); + } + + @Test + public void testPlainAuth_connectionRefused() throws Exception { + assertMemoryLeak(() -> assertThrows("could not connect", + Sender.builder(Sender.Transport.TCP) + .enableAuth("foo").authToken(AUTH_TOKEN_KEY1).address(LOCALHOST + ":19003"))); + } + + @Test + public void testPlainOldTokenNotSupportedForHttpProtocol() throws Exception { + assertMemoryLeak(() -> assertThrows("old token authentication is not supported for HTTP protocol", + Sender.builder(Sender.Transport.HTTP).address("localhost:9000").enableAuth("key").authToken(AUTH_TOKEN_KEY1))); + } + + @Test + public void testPlain_connectionRefused() throws Exception { + assertMemoryLeak(() -> assertThrows("could not connect", + Sender.builder(Sender.Transport.TCP).address(LOCALHOST + ":19003"))); + } + + @Test + public void testPortDoubleSet_firstAddressThenPort() throws Exception { + assertMemoryLeak(() -> assertThrows("mismatch", + Sender.builder(Sender.Transport.TCP).address(LOCALHOST + ":9000").port(9000))); + } + + @Test + public void testPortDoubleSet_firstPortThenAddress() throws Exception { + assertMemoryLeak(() -> assertThrows("mismatch", + Sender.builder(Sender.Transport.TCP).port(9000).address(LOCALHOST + ":9000"))); + } + + @Test + public void testPortDoubleSet_firstPortThenPort() throws Exception { + assertMemoryLeak(() -> assertThrows("questdb server address not set", + Sender.builder(Sender.Transport.TCP).port(9000).port(9000))); + } + + @Test + public void testSmallMaxNameLen() throws Exception { + assertMemoryLeak(() -> assertThrows("max_name_len must be at least 16 bytes [max_name_len=10]", + () -> Sender.builder(Sender.Transport.TCP).maxNameLength(10))); + } + + @Test + public void testTlsDoubleSet() throws Exception { + assertMemoryLeak(() -> assertThrows("already enabled", + () -> Sender.builder(Sender.Transport.TCP).enableTls().enableTls())); + } + + @Test + public void testTlsValidationDisabledButTlsNotEnabled() throws Exception { + assertMemoryLeak(() -> assertThrows("TLS was not enabled", + Sender.builder(Sender.Transport.TCP) + .advancedTls().disableCertificateValidation() + .address(LOCALHOST))); + } + + @Test + public void testTlsValidationDisabledDoubleSet() throws Exception { + assertMemoryLeak(() -> assertThrows("TLS validation was already disabled", + () -> Sender.builder(Sender.Transport.TCP) + .advancedTls().disableCertificateValidation() + .advancedTls().disableCertificateValidation())); + } + + @Test + public void testTls_connectionRefused() throws Exception { + assertMemoryLeak(() -> assertThrows("could not connect", + Sender.builder(Sender.Transport.TCP).enableTls().address(LOCALHOST + ":19003"))); + } + + @Test + public void testUsernamePasswordAuthNotSupportedForTcp() throws Exception { + assertMemoryLeak(() -> assertThrows("username/password authentication is not supported for TCP protocol", + Sender.builder(Sender.Transport.TCP).address(LOCALHOST).httpUsernamePassword("foo", "bar"))); + } + + private static void assertConfStrError(String conf, String expectedError) { + try { + try (Sender ignored = Sender.fromConfig(conf)) { + fail("should fail with bad conf string"); + } + } catch (LineSenderException e) { + TestUtils.assertContains(e.getMessage(), expectedError); + } + } + + private static void assertConfStrOk(String... params) { + StringBuilder sb = new StringBuilder(); + sb.append("http").append("::"); + shuffle(params); + for (int i = 0; i < params.length; i++) { + sb.append(params[i]).append(";"); + } + assertConfStrOk(sb.toString()); + } + + private static void assertConfStrOk(String conf) { + Sender.fromConfig(conf).close(); + } + + private static void assertThrows(String expectedSubstring, Sender.LineSenderBuilder builder) { + assertThrows(expectedSubstring, builder::build); + } + + private static void assertThrows(String expectedSubstring, Runnable action) { + try { + action.run(); + fail("Expected LineSenderException containing '" + expectedSubstring + "'"); + } catch (LineSenderException e) { + TestUtils.assertContains(e.getMessage(), expectedSubstring); + } + } + + private static void shuffle(String[] input) { + for (int i = 0; i < input.length; i++) { + int j = (int) (Math.random() * input.length); + String tmp = input[i]; + input[i] = input[j]; + input[j] = tmp; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/AbstractLineTcpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/AbstractLineTcpSenderTest.java deleted file mode 100644 index 5d426e8..0000000 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/AbstractLineTcpSenderTest.java +++ /dev/null @@ -1,73 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test.cutlass.line.tcp; - -import io.questdb.client.Sender; -import io.questdb.client.cutlass.auth.AuthUtils; -import io.questdb.client.std.Numbers; -import io.questdb.client.test.cutlass.line.AbstractLineSenderTest; - -import java.security.PrivateKey; -import java.util.function.Consumer; - -/** - * Base class for TCP sender integration tests. - * Provides helper methods for creating TCP senders and managing test tables. - */ -public abstract class AbstractLineTcpSenderTest extends AbstractLineSenderTest { - protected static final String AUTH_KEY_ID1 = "testUser1"; - protected final static String AUTH_KEY_ID2_INVALID = "invalid"; - protected final static int HOST = Numbers.parseIPv4("127.0.0.1"); - protected static final Consumer SET_TABLE_NAME_ACTION = s -> s.table("test_mytable"); - protected final static String TOKEN = "UvuVb1USHGRRT08gEnwN2zGZrvM4MsLQ5brgF6SVkAw="; - protected final static PrivateKey AUTH_PRIVATE_KEY1 = AuthUtils.toPrivateKey(TOKEN); - - /** - * Get whether the ILP TCP protocol is authenticated. - */ - protected static boolean getIlpTcpAuthEnabled() { - return getConfigBool("QUESTDB_ILP_TCP_AUTH_ENABLE", "questdb.ilp.tcp.auth.enable", false); - } - - /** - * Get whether the ILP TCP protocol is secure (TLS). - */ - protected static boolean getIlpTcpTlsEnabled() { - return getConfigBool("QUESTDB_ILP_TCP_TLS_ENABLE", "questdb.ilp.tcp.tls.enable", false); - } - - /** - * Create a TCP sender with specified protocol version. - * - * @param protocolVersion the ILP protocol version (V1, V2, or V3) - */ - protected Sender createTcpSender(int protocolVersion) { - return Sender.builder(Sender.Transport.TCP) - .address(getQuestDbHost()) - .port(getIlpTcpPort()) - .protocolVersion(protocolVersion) - .build(); - } -} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/LineTcpAuthSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/LineTcpAuthSenderTest.java deleted file mode 100644 index 3a96303..0000000 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/LineTcpAuthSenderTest.java +++ /dev/null @@ -1,151 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test.cutlass.line.tcp; - -import io.questdb.client.Sender; -import io.questdb.client.cutlass.line.AbstractLineTcpSender; -import io.questdb.client.cutlass.line.LineSenderException; -import io.questdb.client.cutlass.line.LineTcpSenderV2; -import io.questdb.client.std.datetime.microtime.Micros; -import org.junit.Assume; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.time.temporal.ChronoUnit; - -import static io.questdb.client.Sender.PROTOCOL_VERSION_V2; -import static io.questdb.client.Sender.Transport; -import static io.questdb.client.test.tools.TestUtils.assertContains; -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.junit.Assert.fail; - -/** - * Tests for LineTcpSender. - *

    - * Unit tests use DummyLineChannel/ByteChannel (no server needed). - * Integration tests use external QuestDB via AbstractLineTcpSenderTest - * infrastructure. - */ -public class LineTcpAuthSenderTest extends AbstractLineTcpSenderTest { - @BeforeClass - public static void setUpStatic() { - AbstractLineTcpSenderTest.setUpStatic(); - Assume.assumeTrue(getIlpTcpAuthEnabled()); - } - - @Test - public void testAuthSuccess() throws Exception { - useTable("test_auth_success"); - - try (AbstractLineTcpSender sender = LineTcpSenderV2.newSender(HOST, getIlpTcpPort(), 256 * 1024)) { - sender.authenticate(AUTH_KEY_ID1, AUTH_PRIVATE_KEY1); - sender.metric("test_auth_success").field("my int field", 42).$(); - sender.flush(); - } - - assertTableExistsEventually("test_auth_success"); - } - - @Test - public void testAuthWrongKey() { - try (AbstractLineTcpSender sender = LineTcpSenderV2.newSender(HOST, getIlpTcpPort(), 2048)) { - sender.authenticate(AUTH_KEY_ID2_INVALID, AUTH_PRIVATE_KEY1); - // 30 seconds should be enough even on a slow CI server - long deadline = System.nanoTime() + SECONDS.toNanos(30); - while (System.nanoTime() < deadline) { - sender.metric("test_auth_wrong_key").field("my int field", 42).$(); - sender.flush(); - } - fail("Client fail to detected qdb server closed a connection due to wrong credentials"); - } catch (LineSenderException expected) { - // ignored - } - } - - @Test - public void testBuilderAuthSuccess() throws Exception { - useTable("test_builder_auth_success"); - - try (Sender sender = Sender.builder(Transport.TCP) - .address("127.0.0.1:" + getIlpTcpPort()) - .enableAuth(AUTH_KEY_ID1).authToken(TOKEN) - .protocolVersion(PROTOCOL_VERSION_V2) - .build()) { - sender.table("test_builder_auth_success").longColumn("my int field", 42).atNow(); - sender.flush(); - } - - assertTableExistsEventually("test_builder_auth_success"); - } - - @Test - public void testBuilderAuthSuccess_confString() throws Exception { - useTable("test_builder_auth_success_conf_string"); - - try (Sender sender = Sender.fromConfig("tcp::addr=127.0.0.1:" + getIlpTcpPort() + ";user=" + AUTH_KEY_ID1 - + ";token=" + TOKEN + ";protocol_version=2;")) { - sender.table("test_builder_auth_success_conf_string").longColumn("my int field", 42).atNow(); - sender.flush(); - } - - assertTableExistsEventually("test_builder_auth_success_conf_string"); - } - - @Test - public void testConfString() throws Exception { - useTable("test_conf_string"); - - String confString = "tcp::addr=127.0.0.1:" + getIlpTcpPort() + ";user=" + AUTH_KEY_ID1 + ";token=" + TOKEN - + ";protocol_version=2;"; - try (Sender sender = Sender.fromConfig(confString)) { - long tsMicros = Micros.floor("2022-02-25"); - sender.table("test_conf_string") - .longColumn("int_field", 42) - .boolColumn("bool_field", true) - .stringColumn("string_field", "foo") - .doubleColumn("double_field", 42.0) - .timestampColumn("ts_field", tsMicros, ChronoUnit.MICROS) - .at(tsMicros, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually("test_conf_string", 1); - assertSqlEventually( - "int_field\tbool_field\tstring_field\tdouble_field\tts_field\ttimestamp\n" + - "42\ttrue\tfoo\t42.0\t2022-02-25T00:00:00.000000000Z\t2022-02-25T00:00:00.000000000Z\n", - "select int_field, bool_field, string_field, double_field, ts_field, timestamp from test_conf_string"); - } - - @Test - public void testMinBufferSizeWhenAuth() { - int tinyCapacity = 42; - try (AbstractLineTcpSender sender = LineTcpSenderV2.newSender(HOST, getIlpTcpPort(), tinyCapacity)) { - sender.authenticate(AUTH_KEY_ID1, AUTH_PRIVATE_KEY1); - fail(); - } catch (LineSenderException e) { - assertContains(e.getMessage(), "challenge did not fit into buffer"); - } - } -} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/LineTcpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/LineTcpSenderTest.java index 5695a8b..c57bc35 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/LineTcpSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/LineTcpSenderTest.java @@ -25,155 +25,28 @@ package io.questdb.client.test.cutlass.line.tcp; import io.questdb.client.Sender; -import io.questdb.client.cairo.ColumnType; -import io.questdb.client.cutlass.line.AbstractLineTcpSender; import io.questdb.client.cutlass.line.LineChannel; import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.line.LineTcpSenderV2; -import io.questdb.client.cutlass.line.array.DoubleArray; -import io.questdb.client.cutlass.line.tcp.PlainTcpLineChannel; -import io.questdb.client.network.NetworkFacadeImpl; -import io.questdb.client.std.Decimal256; -import io.questdb.client.std.datetime.microtime.Micros; +import io.questdb.client.cutlass.line.LineTcpSenderV3; import io.questdb.client.std.datetime.microtime.MicrosecondClockImpl; -import io.questdb.client.std.datetime.nanotime.Nanos; -import io.questdb.client.test.tools.TestUtils; -import org.junit.Assert; -import org.junit.Assume; -import org.junit.BeforeClass; +import io.questdb.client.test.AbstractTest; import org.junit.Test; -import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.Random; import java.util.function.Consumer; -import static io.questdb.client.Sender.*; import static io.questdb.client.test.tools.TestUtils.assertContains; -import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.Assert.*; /** - * Tests for LineTcpSender. + * Unit tests for LineTcpSender. *

    - * Unit tests use DummyLineChannel/ByteChannel (no server needed). - * Integration tests use external QuestDB via AbstractLineTcpSenderTest - * infrastructure. + * These tests use DummyLineChannel/ByteChannel (no server needed). + * Integration tests have been migrated to core module's LineTcpBootstrapTest. */ -public class LineTcpSenderTest extends AbstractLineTcpSenderTest { - @BeforeClass - public static void setUpStatic() { - AbstractLineTcpSenderTest.setUpStatic(); - Assume.assumeFalse(getIlpTcpAuthEnabled()); - } - - @Test - public void testArrayAtNow() throws Exception { - String table = "test_array_at_now"; - useTable(table); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2); - DoubleArray a1 = new DoubleArray(1, 1, 2, 1).setAll(1)) { - sender.table(table) - .symbol("x", "42i") - .symbol("y", "[6f1.0,2.5,3.0,4.5,5.0]") // ensuring no array parsing for symbol - .longColumn("l1", 23452345) - .doubleArray("a1", a1) - .atNow(); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - } - - @Test - public void testArrayDouble() throws Exception { - String table = "test_array_double"; - useTable(table); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2); - DoubleArray a4 = new DoubleArray(1, 1, 2, 1).setAll(4); - DoubleArray a5 = new DoubleArray(3, 2, 1, 4, 1).setAll(5); - DoubleArray a6 = new DoubleArray(1, 3, 4, 2, 1, 1).setAll(6)) { - long ts = Micros.floor("2025-02-22T00:00:00.000000000Z"); - double[] arr1d = createDoubleArray(5); - double[][] arr2d = createDoubleArray(2, 3); - double[][][] arr3d = createDoubleArray(1, 2, 3); - sender.table(table) - .symbol("x", "42i") - .symbol("y", "[6f1.0,2.5,3.0,4.5,5.0]") // ensuring no array parsing for symbol - .longColumn("l1", 23452345) - .doubleArray("a1", arr1d) - .doubleArray("a2", arr2d) - .doubleArray("a3", arr3d) - .doubleArray("a4", a4) - .doubleArray("a5", a5) - .doubleArray("a6", a6) - .at(ts, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - } - - @Test - public void testAuthWrongKey() throws Exception { - try (AbstractLineTcpSender sender = LineTcpSenderV2.newSender(HOST, getIlpTcpPort(), 2048)) { - sender.authenticate(AUTH_KEY_ID2_INVALID, AUTH_PRIVATE_KEY1); - // 30 seconds should be enough even on a slow CI server - long deadline = System.nanoTime() + SECONDS.toNanos(30); - while (System.nanoTime() < deadline) { - sender.metric("test_auth_wrong_key").field("my int field", 42).$(); - sender.flush(); - } - fail("Client fail to detected qdb server closed a connection due to wrong credentials"); - } catch (LineSenderException expected) { - // ignored - } - } - - @Test - public void testBuilderPlainText_addressWithExplicitIpAndPort() throws Exception { - useTable("test_builder_plain_text_explicit_ip_port"); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - sender.table("test_builder_plain_text_explicit_ip_port").longColumn("my int field", 42).atNow(); - sender.flush(); - } - - assertTableExistsEventually("test_builder_plain_text_explicit_ip_port"); - } - - @Test - public void testBuilderPlainText_addressWithHostnameAndPort() throws Exception { - useTable("test_builder_plain_text_hostname_port"); - - try (Sender sender = Sender.builder(Sender.Transport.TCP) - .address("localhost:" + getIlpTcpPort()) - .protocolVersion(PROTOCOL_VERSION_V2) - .build()) { - sender.table("test_builder_plain_text_hostname_port").longColumn("my int field", 42).atNow(); - sender.flush(); - } - - assertTableExistsEventually("test_builder_plain_text_hostname_port"); - } - - @Test - public void testBuilderPlainText_addressWithIpAndPort() throws Exception { - useTable("test_builder_plain_text_ip_port"); - - String address = "127.0.0.1:" + getIlpTcpPort(); - try (Sender sender = Sender.builder(Sender.Transport.TCP) - .address(address) - .protocolVersion(PROTOCOL_VERSION_V2) - .build()) { - sender.table("test_builder_plain_text_ip_port").longColumn("my int field", 42).atNow(); - sender.flush(); - } - - assertTableExistsEventually("test_builder_plain_text_ip_port"); - } +public class LineTcpSenderTest extends AbstractTest { + protected static final Consumer SET_TABLE_NAME_ACTION = s -> s.table("test_mytable"); @Test public void testCannotStartNewRowBeforeClosingTheExistingAfterValidationError() { @@ -199,45 +72,12 @@ public void testCannotStartNewRowBeforeClosingTheExistingAfterValidationError() @Test public void testCloseIdempotent() { DummyLineChannel channel = new DummyLineChannel(); - AbstractLineTcpSender sender = new LineTcpSenderV2(channel, 1000, 127); + LineTcpSenderV2 sender = new LineTcpSenderV2(channel, 1000, 127); sender.close(); sender.close(); assertEquals(1, channel.closeCounter); } - @Test - public void testCloseImpliesFlush() throws Exception { - useTable("test_close_implies_flush"); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - sender.table("test_close_implies_flush").longColumn("my int field", 42).atNow(); - } - - assertTableExistsEventually("test_close_implies_flush"); - } - - @Test - public void testConfString_autoFlushBytes() throws Exception { - useTable("test_conf_string_auto_flush_bytes"); - - String confString = "tcp::addr=localhost:" + getIlpTcpPort() + ";auto_flush_bytes=1;protocol_version=2;"; // the - // minimal - // allowed - // buffer - // size - try (Sender sender = Sender.fromConfig(confString)) { - // just 2 rows must be enough to trigger flush - // why not 1? the first byte of the 2nd row will flush the last byte of the 1st - // row - sender.table("test_conf_string_auto_flush_bytes").longColumn("my int field", 42).atNow(); - sender.table("test_conf_string_auto_flush_bytes").longColumn("my int field", 42).atNow(); - - // make sure to assert before closing the Sender - // since the Sender will always flush on close - assertTableExistsEventually("test_conf_string_auto_flush_bytes"); - } - } - @Test public void testControlCharInColumnName() { assertControlCharacterException(); @@ -248,1049 +88,68 @@ public void testControlCharInTableName() { assertControlCharacterException(); } - @Test - public void testCreateTimestampColumnsWithDesignatedInstantV1() throws Exception { - testCreateTimestampColumns(Nanos.floor("2025-11-20T10:55:24.123123123Z"), null, PROTOCOL_VERSION_V1, - new int[]{ColumnType.TIMESTAMP, ColumnType.TIMESTAMP, ColumnType.TIMESTAMP}, - "1.111\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123000000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.123123000Z"); - } - - @Test - public void testCreateTimestampColumnsWithDesignatedInstantV2() throws Exception { - testCreateTimestampColumns(Nanos.floor("2025-11-20T10:55:24.123123123Z"), null, PROTOCOL_VERSION_V2, - new int[]{ColumnType.TIMESTAMP_NANO, ColumnType.TIMESTAMP_NANO, ColumnType.TIMESTAMP_NANO}, - "1.111\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123000000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.123123000Z"); - } - - @Test - public void testCreateTimestampColumnsWithDesignatedMicrosV1() throws Exception { - testCreateTimestampColumns(Micros.floor("2025-11-20T10:55:24.123456000Z"), ChronoUnit.MICROS, - PROTOCOL_VERSION_V1, - new int[]{ColumnType.TIMESTAMP, ColumnType.TIMESTAMP, ColumnType.TIMESTAMP}, - "1.111\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123000000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.123456000Z"); - } - - @Test - public void testCreateTimestampColumnsWithDesignatedMicrosV2() throws Exception { - testCreateTimestampColumns(Micros.floor("2025-11-20T10:55:24.123456000Z"), ChronoUnit.MICROS, - PROTOCOL_VERSION_V2, - new int[]{ColumnType.TIMESTAMP_NANO, ColumnType.TIMESTAMP_NANO, ColumnType.TIMESTAMP}, - "1.111\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123000000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.123456000Z"); - } - - @Test - public void testCreateTimestampColumnsWithDesignatedMillisV1() throws Exception { - testCreateTimestampColumns(Micros.floor("2025-11-20T10:55:24.123456000Z") / 1000, ChronoUnit.MILLIS, - PROTOCOL_VERSION_V1, - new int[]{ColumnType.TIMESTAMP, ColumnType.TIMESTAMP, ColumnType.TIMESTAMP}, - "1.111\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123000000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.123000000Z"); - } - - @Test - public void testCreateTimestampColumnsWithDesignatedMillisV2() throws Exception { - testCreateTimestampColumns(Micros.floor("2025-11-20T10:55:24.123456000Z") / 1000, ChronoUnit.MILLIS, - PROTOCOL_VERSION_V2, - new int[]{ColumnType.TIMESTAMP_NANO, ColumnType.TIMESTAMP_NANO, ColumnType.TIMESTAMP}, - "1.111\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123000000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.123000000Z"); - } - - @Test - public void testCreateTimestampColumnsWithDesignatedNanosV1() throws Exception { - testCreateTimestampColumns(Nanos.floor("2025-11-20T10:55:24.123456789Z"), ChronoUnit.NANOS, PROTOCOL_VERSION_V1, - new int[]{ColumnType.TIMESTAMP, ColumnType.TIMESTAMP, ColumnType.TIMESTAMP}, - "1.111\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123000000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.123456000Z"); - } - - @Test - public void testCreateTimestampColumnsWithDesignatedNanosV2() throws Exception { - testCreateTimestampColumns(Nanos.floor("2025-11-20T10:55:24.123456789Z"), ChronoUnit.NANOS, PROTOCOL_VERSION_V2, - new int[]{ColumnType.TIMESTAMP_NANO, ColumnType.TIMESTAMP_NANO, ColumnType.TIMESTAMP_NANO}, - "1.111\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123000000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.123456000Z"); - } - - @Test - public void testDecimalDefaultValuesWithoutWal() throws Exception { - useTable("test_decimal_default_values_without_wal"); - execute( - "CREATE TABLE test_decimal_default_values_without_wal (\n" + - " dec8 DECIMAL(2, 0),\n" + - " dec16 DECIMAL(4, 1),\n" + - " dec32 DECIMAL(8, 2),\n" + - " dec64 DECIMAL(16, 4),\n" + - " dec128 DECIMAL(34, 8),\n" + - " dec256 DECIMAL(64, 16),\n" + - " value INT,\n" + - " ts TIMESTAMP\n" + - ") TIMESTAMP(ts) PARTITION BY DAY BYPASS WAL\n"); - - assertTableExistsEventually("test_decimal_default_values_without_wal"); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V3)) { - sender.table("test_decimal_default_values_without_wal") - .longColumn("value", 1) - .at(100_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually("test_decimal_default_values_without_wal", 1); - assertSqlEventually( - "dec8\tdec16\tdec32\tdec64\tdec128\tdec256\tvalue\tts\n" + - "null\tnull\tnull\tnull\tnull\tnull\t1\t1970-01-01T00:00:00.100000000Z\n", - "select dec8, dec16, dec32, dec64, dec128, dec256, value, ts from test_decimal_default_values_without_wal"); - } - - @Test - public void testDouble_edgeValues() throws Exception { - useTable("test_double_edge_values"); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - long ts = Micros.floor("2022-02-25"); - sender.table("test_double_edge_values") - .doubleColumn("negative_inf", Double.NEGATIVE_INFINITY) - .doubleColumn("positive_inf", Double.POSITIVE_INFINITY) - .doubleColumn("nan", Double.NaN) - .doubleColumn("max_value", Double.MAX_VALUE) - .doubleColumn("min_value", Double.MIN_VALUE) - .at(ts, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually("test_double_edge_values", 1); - assertSqlEventually( - "negative_inf\tpositive_inf\tnan\tmax_value\tmin_value\ttimestamp\n" + - "null\tnull\tnull\t1.7976931348623157E308\t4.9E-324\t2022-02-25T00:00:00.000000000Z\n", - "select negative_inf, positive_inf, nan, max_value, min_value, timestamp from test_double_edge_values"); - } - - @Test - public void testExplicitTimestampColumnIndexIsCleared() throws Exception { - useTable("test_explicit_ts_col_idx_cleared_poison"); - useTable("test_explicit_ts_col_idx_cleared_victim"); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - long ts = Micros.floor("2022-02-25"); - // the poison table sets the timestamp column index explicitly - sender.table("test_explicit_ts_col_idx_cleared_poison") - .stringColumn("str_col1", "str_col1") - .stringColumn("str_col2", "str_col2") - .stringColumn("str_col3", "str_col3") - .stringColumn("str_col4", "str_col4") - .timestampColumn("timestamp", ts, ChronoUnit.MICROS) - .at(ts, ChronoUnit.MICROS); - sender.flush(); - assertTableSizeEventually("test_explicit_ts_col_idx_cleared_poison", 1); - - // the victim table does not set the timestamp column index explicitly - sender.table("test_explicit_ts_col_idx_cleared_victim") - .stringColumn("str_col1", "str_col1") - .at(ts, ChronoUnit.MICROS); - sender.flush(); - assertTableSizeEventually("test_explicit_ts_col_idx_cleared_victim", 1); - } - } - - @Test - public void testInsertBadStringIntoUuidColumn() throws Exception { - testValueCannotBeInsertedToUuidColumn("test_insert_bad_string_into_uuid_column", "totally not a uuid"); - } - - @Test - public void testInsertBinaryToOtherColumns() throws Exception { - useTable("test_insert_binary_to_other_columns"); - execute( - "CREATE TABLE test_insert_binary_to_other_columns (\n" + - " x SYMBOL,\n" + - " y VARCHAR,\n" + - " a1 DOUBLE,\n" + - " timestamp TIMESTAMP\n" + - ") TIMESTAMP(timestamp) PARTITION BY YEAR BYPASS WAL\n"); - - assertTableExistsEventually("test_insert_binary_to_other_columns"); - - // send text double to symbol column - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V1)) { - sender.table("test_insert_binary_to_other_columns") - .doubleColumn("x", 9999.0) - .stringColumn("y", "ystr") - .doubleColumn("a1", 1) - .at(100000000000L, ChronoUnit.MICROS); - sender.flush(); - } - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - // insert binary double to symbol column - sender.table("test_insert_binary_to_other_columns") - .doubleColumn("x", 10000.0) - .stringColumn("y", "ystr") - .doubleColumn("a1", 1) - .at(100000000001L, ChronoUnit.MICROS); - sender.flush(); - - // insert binary double to string column (should be rejected) - sender.table("test_insert_binary_to_other_columns") - .symbol("x", "x1") - .doubleColumn("y", 9999.0) - .doubleColumn("a1", 1) - .at(100000000000L, ChronoUnit.MICROS); - sender.flush(); - // insert string to double column (should be rejected) - sender.table("test_insert_binary_to_other_columns") - .symbol("x", "x1") - .stringColumn("y", "ystr") - .stringColumn("a1", "11.u") - .at(100000000000L, ChronoUnit.MICROS); - sender.flush(); - // insert array column to double (should be rejected) - sender.table("test_insert_binary_to_other_columns") - .symbol("x", "x1") - .stringColumn("y", "ystr") - .doubleArray("a1", new double[]{1.0, 2.0}) - .at(100000000000L, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually("test_insert_binary_to_other_columns", 2); - assertSqlEventually( - "x\ty\ta1\ttimestamp\n" + - "9999.0\tystr\t1.0\t1970-01-02T03:46:40.000000000Z\n" + - "10000.0\tystr\t1.0\t1970-01-02T03:46:40.000001000Z\n", - "select x, y, a1, timestamp from test_insert_binary_to_other_columns order by timestamp"); - } - - @Test - public void testInsertDecimalTextFormatBasic() throws Exception { - String tableName = "test_decimal_text_format_basic"; - useTable(tableName); - execute( - "CREATE TABLE test_decimal_text_format_basic (\n" + - " price DECIMAL(10, 2),\n" + - " quantity DECIMAL(15, 4),\n" + - " rate DECIMAL(8, 5),\n" + - " timestamp TIMESTAMP\n" + - ") TIMESTAMP(timestamp) PARTITION BY NONE BYPASS WAL\n"); - - assertTableExistsEventually(tableName); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V3)) { - // Basic positive decimal - sender.table(tableName) - .decimalColumn("price", "123.45") - .decimalColumn("quantity", "100.0000") - .decimalColumn("rate", "0.12345") - .at(100000000000L, ChronoUnit.MICROS); - - // Negative decimal - sender.table(tableName) - .decimalColumn("price", "-45.67") - .decimalColumn("quantity", "-10.5000") - .decimalColumn("rate", "-0.00001") - .at(100000000001L, ChronoUnit.MICROS); - - // Small values - sender.table(tableName) - .decimalColumn("price", "0.01") - .decimalColumn("quantity", "0.0001") - .decimalColumn("rate", "0.00000") - .at(100000000002L, ChronoUnit.MICROS); - - // Integer strings (no decimal point) - sender.table(tableName) - .decimalColumn("price", "999") - .decimalColumn("quantity", "42") - .decimalColumn("rate", "1") - .at(100000000003L, ChronoUnit.MICROS); - - sender.flush(); - } - - assertTableSizeEventually(tableName, 4); - assertSqlEventually( - "price\tquantity\trate\ttimestamp\n" + - "123.45\t100.0000\t0.12345\t1970-01-02T03:46:40.000000000Z\n" + - "-45.67\t-10.5000\t-0.00001\t1970-01-02T03:46:40.000001000Z\n" + - "0.01\t0.0001\t0.00000\t1970-01-02T03:46:40.000002000Z\n" + - "999.00\t42.0000\t1.00000\t1970-01-02T03:46:40.000003000Z\n", - "select price, quantity, rate, timestamp from " + tableName + " order by timestamp"); - } - - @Test - public void testInsertDecimalTextFormatEdgeCases() throws Exception { - String tableName = "test_decimal_text_format_edge_cases"; - useTable(tableName); - execute( - "CREATE TABLE test_decimal_text_format_edge_cases (\n" + - " value DECIMAL(20, 10),\n" + - " timestamp TIMESTAMP\n" + - ") TIMESTAMP(timestamp) PARTITION BY NONE BYPASS WAL\n"); - - assertTableExistsEventually(tableName); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V3)) { - // Explicit positive sign - sender.table(tableName) - .decimalColumn("value", "+123.456") - .at(100000000000L, ChronoUnit.MICROS); - - // Leading zeros - sender.table(tableName) - .decimalColumn("value", "000123.450000") - .at(100000000001L, ChronoUnit.MICROS); - - // Very small value - sender.table(tableName) - .decimalColumn("value", "0.0000000001") - .at(100000000002L, ChronoUnit.MICROS); - - // Zero with decimal point - sender.table(tableName) - .decimalColumn("value", "0.0") - .at(100000000003L, ChronoUnit.MICROS); - - // Just zero - sender.table(tableName) - .decimalColumn("value", "0") - .at(100000000004L, ChronoUnit.MICROS); - - sender.flush(); - } - - assertTableSizeEventually(tableName, 5); - assertSqlEventually( - "value\ttimestamp\n" + - "123.4560000000\t1970-01-02T03:46:40.000000000Z\n" + - "123.4500000000\t1970-01-02T03:46:40.000001000Z\n" + - "0.0000000001\t1970-01-02T03:46:40.000002000Z\n" + - "0.0000000000\t1970-01-02T03:46:40.000003000Z\n" + - "0.0000000000\t1970-01-02T03:46:40.000004000Z\n", - "select value, timestamp from " + tableName + " order by timestamp"); - } - - @Test - public void testInsertDecimalTextFormatEquivalence() throws Exception { - String tableName = "test_decimal_text_format_equivalence"; - useTable(tableName); - execute( - "CREATE TABLE test_decimal_text_format_equivalence (\n" + - " text_format DECIMAL(10, 3),\n" + - " binary_format DECIMAL(10, 3),\n" + - " timestamp TIMESTAMP\n" + - ") TIMESTAMP(timestamp) PARTITION BY NONE BYPASS WAL\n"); - - assertTableExistsEventually(tableName); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V3)) { - // Test various values sent via both text and binary formats - sender.table(tableName) - .decimalColumn("text_format", "123.450") - .decimalColumn("binary_format", Decimal256.fromLong(123450, 3)) - .at(100000000000L, ChronoUnit.MICROS); - - sender.table(tableName) - .decimalColumn("text_format", "-45.670") - .decimalColumn("binary_format", Decimal256.fromLong(-45670, 3)) - .at(100000000001L, ChronoUnit.MICROS); - - sender.table(tableName) - .decimalColumn("text_format", "0.001") - .decimalColumn("binary_format", Decimal256.fromLong(1, 3)) - .at(100000000002L, ChronoUnit.MICROS); - - sender.flush(); - } - - assertTableSizeEventually(tableName, 3); - assertSqlEventually( - "text_format\tbinary_format\ttimestamp\n" + - "123.450\t123.450\t1970-01-02T03:46:40.000000000Z\n" + - "-45.670\t-45.670\t1970-01-02T03:46:40.000001000Z\n" + - "0.001\t0.001\t1970-01-02T03:46:40.000002000Z\n", - "select text_format, binary_format, timestamp from " + tableName + " order by timestamp"); - } - @Test public void testInsertDecimalTextFormatInvalid() throws Exception { - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V3)) { + try (Sender sender = new LineTcpSenderV3(new DummyLineChannel(), 4096, 127)) { sender.table("test"); // Test invalid characters try { sender.decimalColumn("value", "abc"); - Assert.fail("Letters should throw exception"); + fail("Letters should throw exception"); } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "Failed to parse sent decimal value"); + assertContains(e.getMessage(), "Failed to parse sent decimal value"); } // Test multiple dots try { sender.decimalColumn("value", "12.34.56"); - Assert.fail("Multiple dots should throw exception"); + fail("Multiple dots should throw exception"); } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "Failed to parse sent decimal value"); + assertContains(e.getMessage(), "Failed to parse sent decimal value"); } // Test multiple signs try { sender.decimalColumn("value", "+-123"); - Assert.fail("Multiple signs should throw exception"); + fail("Multiple signs should throw exception"); } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "Failed to parse sent decimal value"); + assertContains(e.getMessage(), "Failed to parse sent decimal value"); } // Test special characters try { sender.decimalColumn("value", "12$34"); - Assert.fail("Special characters should throw exception"); + fail("Special characters should throw exception"); } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "Failed to parse sent decimal value"); + assertContains(e.getMessage(), "Failed to parse sent decimal value"); } // Test empty decimal try { sender.decimalColumn("value", ""); - Assert.fail("Empty string should throw exception"); + fail("Empty string should throw exception"); } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "Failed to parse sent decimal value"); + assertContains(e.getMessage(), "Failed to parse sent decimal value"); } // Test invalid exponent try { sender.decimalColumn("value", "1.23eABC"); - Assert.fail("Invalid exponent should throw exception"); + fail("Invalid exponent should throw exception"); } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "Failed to parse sent decimal value"); + assertContains(e.getMessage(), "Failed to parse sent decimal value"); } // Test incomplete exponent try { sender.decimalColumn("value", "1.23e"); - Assert.fail("Incomplete exponent should throw exception"); + fail("Incomplete exponent should throw exception"); } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "Failed to parse sent decimal value"); + assertContains(e.getMessage(), "Failed to parse sent decimal value"); } } } - @Test - public void testInsertDecimalTextFormatPrecisionOverflow() throws Exception { - String tableName = "test_decimal_text_format_precision_overflow"; - useTable(tableName); - execute( - "CREATE TABLE test_decimal_text_format_precision_overflow (\n" + - " x DECIMAL(6, 3),\n" + - " timestamp TIMESTAMP\n" + - ") TIMESTAMP(timestamp) PARTITION BY NONE BYPASS WAL\n"); - - assertTableExistsEventually(tableName); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V3)) { - // Value that exceeds column precision (6 digits total, 3 after decimal) - // 1000.000 has 7 digits precision, should be rejected - sender.table(tableName) - .decimalColumn("x", "1000.000") - .at(100000000000L, ChronoUnit.MICROS); - - // Another value that exceeds precision - sender.table(tableName) - .decimalColumn("x", "12345.678") - .at(100000000001L, ChronoUnit.MICROS); - - sender.flush(); - } - - assertSqlEventually( - "x\ttimestamp\n", - "select x, timestamp from " + tableName); - } - - @Test - public void testInsertDecimalTextFormatScientificNotation() throws Exception { - String tableName = "test_decimal_text_format_scientific_notation"; - useTable(tableName); - execute( - "CREATE TABLE test_decimal_text_format_scientific_notation (\n" + - " large DECIMAL(15, 2),\n" + - " small DECIMAL(20, 15),\n" + - " timestamp TIMESTAMP\n" + - ") TIMESTAMP(timestamp) PARTITION BY NONE BYPASS WAL\n"); - - assertTableExistsEventually(tableName); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V3)) { - // Scientific notation with positive exponent - sender.table(tableName) - .decimalColumn("large", "1.23e5") - .decimalColumn("small", "1.23e-10") - .at(100000000000L, ChronoUnit.MICROS); - - // Scientific notation with uppercase E - sender.table(tableName) - .decimalColumn("large", "4.56E3") - .decimalColumn("small", "4.56E-8") - .at(100000000001L, ChronoUnit.MICROS); - - // Negative value with scientific notation - sender.table(tableName) - .decimalColumn("large", "-9.99e2") - .decimalColumn("small", "-1.5e-12") - .at(100000000002L, ChronoUnit.MICROS); - - sender.flush(); - } - - assertTableSizeEventually(tableName, 3); - assertSqlEventually( - "large\tsmall\ttimestamp\n" + - "123000.00\t0.000000000123000\t1970-01-02T03:46:40.000000000Z\n" + - "4560.00\t0.000000045600000\t1970-01-02T03:46:40.000001000Z\n" + - "-999.00\t-0.000000000001500\t1970-01-02T03:46:40.000002000Z\n", - "select large, small, timestamp from " + tableName + " order by timestamp"); - } - - @Test - public void testInsertDecimalTextFormatTrailingZeros() throws Exception { - String tableName = "test_decimal_text_format_trailing_zeros"; - useTable(tableName); - execute( - "CREATE TABLE test_decimal_text_format_trailing_zeros (\n" + - " value1 DECIMAL(10, 3),\n" + - " value2 DECIMAL(12, 5),\n" + - " timestamp TIMESTAMP\n" + - ") TIMESTAMP(timestamp) PARTITION BY NONE BYPASS WAL\n"); - - assertTableExistsEventually(tableName); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V3)) { - // Trailing zeros should be preserved in scale - sender.table(tableName) - .decimalColumn("value1", "100.000") - .decimalColumn("value2", "50.00000") - .at(100000000000L, ChronoUnit.MICROS); - - sender.table(tableName) - .decimalColumn("value1", "1.200") - .decimalColumn("value2", "0.12300") - .at(100000000001L, ChronoUnit.MICROS); - - sender.table(tableName) - .decimalColumn("value1", "0.100") - .decimalColumn("value2", "0.00100") - .at(100000000002L, ChronoUnit.MICROS); - - sender.flush(); - } - - assertTableSizeEventually(tableName, 3); - assertSqlEventually( - "value1\tvalue2\ttimestamp\n" + - "100.000\t50.00000\t1970-01-02T03:46:40.000000000Z\n" + - "1.200\t0.12300\t1970-01-02T03:46:40.000001000Z\n" + - "0.100\t0.00100\t1970-01-02T03:46:40.000002000Z\n", - "select value1, value2, timestamp from " + tableName + " order by timestamp"); - } - - @Test - public void testInsertDecimals() throws Exception { - String tableName = "test_insert_decimals"; - useTable(tableName); - execute( - "CREATE TABLE test_insert_decimals (\n" + - " a DECIMAL(9, 0),\n" + - " b DECIMAL(9, 3),\n" + - " timestamp TIMESTAMP\n" + - ") TIMESTAMP(timestamp) PARTITION BY NONE BYPASS WAL\n"); - - assertTableExistsEventually(tableName); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V3)) { - sender.table(tableName) - .decimalColumn("a", Decimal256.fromLong(12345, 0)) - .decimalColumn("b", Decimal256.fromLong(12345, 2)) - .at(100000000000L, ChronoUnit.MICROS); - - // Decimal without rescale - sender.table(tableName) - .decimalColumn("a", Decimal256.NULL_VALUE) - .decimalColumn("b", Decimal256.fromLong(123456, 3)) - .at(100000000001L, ChronoUnit.MICROS); - - // Integers -> Decimal - sender.table(tableName) - .longColumn("a", 42) - .longColumn("b", 42) - .at(100000000002L, ChronoUnit.MICROS); - - // Strings -> Decimal without rescale - sender.table(tableName) - .stringColumn("a", "42") - .stringColumn("b", "42.123") - .at(100000000003L, ChronoUnit.MICROS); - - // Strings -> Decimal with rescale - sender.table(tableName) - .stringColumn("a", "42.0") - .stringColumn("b", "42.1") - .at(100000000004L, ChronoUnit.MICROS); - - // Doubles -> Decimal - sender.table(tableName) - .doubleColumn("a", 42d) - .doubleColumn("b", 42.1d) - .at(100000000005L, ChronoUnit.MICROS); - - // NaN/Inf Doubles -> Decimal - sender.table(tableName) - .doubleColumn("a", Double.NaN) - .doubleColumn("b", Double.POSITIVE_INFINITY) - .at(100000000006L, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(tableName, 7); - assertSqlEventually( - "a\tb\ttimestamp\n" + - "12345\t123.450\t1970-01-02T03:46:40.000000000Z\n" + - "null\t123.456\t1970-01-02T03:46:40.000001000Z\n" + - "42\t42.000\t1970-01-02T03:46:40.000002000Z\n" + - "42\t42.123\t1970-01-02T03:46:40.000003000Z\n" + - "42\t42.100\t1970-01-02T03:46:40.000004000Z\n" + - "42\t42.100\t1970-01-02T03:46:40.000005000Z\n" + - "null\tnull\t1970-01-02T03:46:40.000006000Z\n", - "select a, b, timestamp from " + tableName + " order by timestamp"); - } - - @Test - public void testInsertInvalidDecimals() throws Exception { - String tableName = "test_invalid_decimal_test"; - useTable(tableName); - execute( - "CREATE TABLE test_invalid_decimal_test (\n" + - " x DECIMAL(6, 3),\n" + - " y DECIMAL(76, 73),\n" + - " timestamp TIMESTAMP\n" + - ") TIMESTAMP(timestamp) PARTITION BY NONE BYPASS WAL\n"); - - assertTableExistsEventually(tableName); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V3)) { - // Integers out of bound (with scaling, 1234 becomes 1234.000 which have a - // precision of 7). - sender.table(tableName) - .longColumn("x", 1234) - .at(100000000000L, ChronoUnit.MICROS); - - // Integers overbound during the rescale process. - sender.table(tableName) - .longColumn("y", 12345) - .at(100000000001L, ChronoUnit.MICROS); - - // Floating points with a scale greater than expected. - sender.table(tableName) - .doubleColumn("x", 1.2345d) - .at(100000000002L, ChronoUnit.MICROS); - - // Floating points with a precision greater than expected. - sender.table(tableName) - .doubleColumn("x", 12345.678d) - .at(100000000003L, ChronoUnit.MICROS); - - // String that is not a valid decimal. - sender.table(tableName) - .stringColumn("x", "abc") - .at(100000000004L, ChronoUnit.MICROS); - - // String that has a too big precision. - sender.table(tableName) - .stringColumn("x", "1E8") - .at(100000000005L, ChronoUnit.MICROS); - - // Decimal with a too big precision. - sender.table(tableName) - .decimalColumn("x", Decimal256.fromLong(12345678, 3)) - .at(100000000006L, ChronoUnit.MICROS); - - // Decimal with a too big precision when scaled. - sender.table(tableName) - .decimalColumn("y", Decimal256.fromLong(12345, 0)) - .at(100000000007L, ChronoUnit.MICROS); - sender.flush(); - - // Decimal loosing precision - sender.table(tableName) - .decimalColumn("x", Decimal256.fromLong(123456, 4)) - .at(100000000007L, ChronoUnit.MICROS); - sender.flush(); - } - - assertSqlEventually( - "x\ty\ttimestamp\n", - "select x, y, timestamp from " + tableName); - } - - @Test - public void testInsertLargeArray() throws Exception { - String tableName = "test_arr_large_test"; - useTable(tableName); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - double[] arr = createDoubleArray(10_000_000); - sender.table(tableName) - .doubleArray("arr", arr) - .at(100000000000L, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(tableName, 1); - } - - @Test - public void testInsertNonAsciiStringAndUuid() throws Exception { - // this is to check that a non-ASCII string will not prevent - // parsing a subsequent UUID - useTable("test_insert_non_ascii_string_and_uuid"); - execute( - "CREATE TABLE test_insert_non_ascii_string_and_uuid (\n" + - " s STRING,\n" + - " u UUID,\n" + - " ts TIMESTAMP\n" + - ") TIMESTAMP(ts) PARTITION BY NONE BYPASS WAL\n"); - - assertTableExistsEventually("test_insert_non_ascii_string_and_uuid"); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - long tsMicros = Micros.floor("2022-02-25"); - sender.table("test_insert_non_ascii_string_and_uuid") - .stringColumn("s", "non-ascii äöü") - .stringColumn("u", "11111111-2222-3333-4444-555555555555") - .at(tsMicros, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually("test_insert_non_ascii_string_and_uuid", 1); - assertSqlEventually( - "s\tu\tts\n" + - "non-ascii äöü\t11111111-2222-3333-4444-555555555555\t2022-02-25T00:00:00.000000000Z\n", - "select s, u, ts from test_insert_non_ascii_string_and_uuid"); - } - - @Test - public void testInsertNonAsciiStringIntoUuidColumn() throws Exception { - // carefully crafted value so when encoded as UTF-8 it has the same byte length - // as a proper UUID - testValueCannotBeInsertedToUuidColumn("test_insert_non_ascii_string_into_uuid_column", - "11111111-1111-1111-1111-1111111111ü"); - } - - @Test - public void testInsertStringIntoUuidColumn() throws Exception { - useTable("test_insert_string_into_uuid_column"); - execute( - "CREATE TABLE test_insert_string_into_uuid_column (\n" + - " u1 UUID,\n" + - " u2 UUID,\n" + - " u3 UUID,\n" + - " ts TIMESTAMP\n" + - ") TIMESTAMP(ts) PARTITION BY NONE BYPASS WAL\n"); - - assertTableExistsEventually("test_insert_string_into_uuid_column"); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - long tsMicros = Micros.floor("2022-02-25"); - sender.table("test_insert_string_into_uuid_column") - .stringColumn("u1", "11111111-1111-1111-1111-111111111111") - // u2 empty -> insert as null - .stringColumn("u3", "33333333-3333-3333-3333-333333333333") - .at(tsMicros, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually("test_insert_string_into_uuid_column", 1); - assertSqlEventually( - "u1\tu3\tts\n" + - "11111111-1111-1111-1111-111111111111\t33333333-3333-3333-3333-333333333333\t2022-02-25T00:00:00.000000000Z\n", - "select u1, u3, ts from test_insert_string_into_uuid_column"); - } - - @Test - public void testInsertTimestampAsInstant() throws Exception { - useTable("test_insert_timestamp_as_instant"); - execute( - "CREATE TABLE test_insert_timestamp_as_instant (\n" + - " ts_col TIMESTAMP,\n" + - " timestamp TIMESTAMP\n" + - ") TIMESTAMP(timestamp) PARTITION BY YEAR BYPASS WAL\n"); - - assertTableExistsEventually("test_insert_timestamp_as_instant"); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - sender.table("test_insert_timestamp_as_instant") - .timestampColumn("ts_col", Instant.parse("2023-02-11T12:30:11.35Z")) - .at(Instant.parse("2022-01-10T20:40:22.54Z")); - sender.flush(); - } - - assertTableSizeEventually("test_insert_timestamp_as_instant", 1); - assertSqlEventually( - "ts_col\ttimestamp\n" + - "2023-02-11T12:30:11.350000000Z\t2022-01-10T20:40:22.540000000Z\n", - "select ts_col, timestamp from test_insert_timestamp_as_instant"); - } - - @Test - public void testInsertTimestampMiscUnits() throws Exception { - useTable("test_insert_timestamp_misc_units"); - execute( - "CREATE TABLE test_insert_timestamp_misc_units (\n" + - " unit STRING,\n" + - " ts TIMESTAMP,\n" + - " timestamp TIMESTAMP\n" + - ") TIMESTAMP(timestamp) PARTITION BY YEAR BYPASS WAL\n"); - - assertTableExistsEventually("test_insert_timestamp_misc_units"); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - long tsMicros = Micros.floor("2023-09-18T12:01:01.01Z"); - sender.table("test_insert_timestamp_misc_units") - .stringColumn("unit", "ns") - .timestampColumn("ts", tsMicros * 1000, ChronoUnit.NANOS) - .at(tsMicros * 1000, ChronoUnit.NANOS); - sender.table("test_insert_timestamp_misc_units") - .stringColumn("unit", "us") - .timestampColumn("ts", tsMicros, ChronoUnit.MICROS) - .at(tsMicros, ChronoUnit.MICROS); - sender.table("test_insert_timestamp_misc_units") - .stringColumn("unit", "ms") - .timestampColumn("ts", tsMicros / 1000, ChronoUnit.MILLIS) - .at(tsMicros / 1000, ChronoUnit.MILLIS); - sender.table("test_insert_timestamp_misc_units") - .stringColumn("unit", "s") - .timestampColumn("ts", tsMicros / Micros.SECOND_MICROS, ChronoUnit.SECONDS) - .at(tsMicros / Micros.SECOND_MICROS, ChronoUnit.SECONDS); - sender.table("test_insert_timestamp_misc_units") - .stringColumn("unit", "m") - .timestampColumn("ts", tsMicros / Micros.MINUTE_MICROS, ChronoUnit.MINUTES) - .at(tsMicros / Micros.MINUTE_MICROS, ChronoUnit.MINUTES); - sender.flush(); - } - - assertTableSizeEventually("test_insert_timestamp_misc_units", 5); - assertSqlEventually( - "unit\tts\ttimestamp\n" + - "m\t2023-09-18T12:01:00.000000000Z\t2023-09-18T12:01:00.000000000Z\n" + - "s\t2023-09-18T12:01:01.000000000Z\t2023-09-18T12:01:01.000000000Z\n" + - "ns\t2023-09-18T12:01:01.010000000Z\t2023-09-18T12:01:01.010000000Z\n" + - "us\t2023-09-18T12:01:01.010000000Z\t2023-09-18T12:01:01.010000000Z\n" + - "ms\t2023-09-18T12:01:01.010000000Z\t2023-09-18T12:01:01.010000000Z\n", - "select unit, ts, timestamp from test_insert_timestamp_misc_units order by timestamp"); - } - - @Test - public void testInsertTimestampNanoOverflow() throws Exception { - useTable("test_insert_timestamp_nano_overflow"); - execute( - "CREATE TABLE test_insert_timestamp_nano_overflow (\n" + - " ts TIMESTAMP,\n" + - " timestamp TIMESTAMP\n" + - ") TIMESTAMP(timestamp) PARTITION BY YEAR BYPASS WAL\n"); - - assertTableExistsEventually("test_insert_timestamp_nano_overflow"); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - long tsMicros = Micros.floor("2323-09-18T12:01:01.011568901Z"); - sender.table("test_insert_timestamp_nano_overflow") - .timestampColumn("ts", tsMicros, ChronoUnit.MICROS) - .at(tsMicros, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually("test_insert_timestamp_nano_overflow", 1); - assertSqlEventually( - "ts\ttimestamp\n" + - "2323-09-18T12:01:01.011568000Z\t2323-09-18T12:01:01.011568000Z\n", - "select ts, timestamp from test_insert_timestamp_nano_overflow"); - } - - @Test - public void testInsertTimestampNanoUnits() throws Exception { - useTable("test_insert_timestamp_nano_units"); - execute( - "CREATE TABLE test_insert_timestamp_nano_units (\n" + - " unit STRING,\n" + - " ts TIMESTAMP,\n" + - " timestamp TIMESTAMP\n" + - ") TIMESTAMP(timestamp) PARTITION BY YEAR BYPASS WAL\n"); - - assertTableExistsEventually("test_insert_timestamp_nano_units"); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - long tsNanos = Micros.floor("2023-09-18T12:01:01.011568901Z") * 1000; - sender.table("test_insert_timestamp_nano_units") - .stringColumn("unit", "ns") - .timestampColumn("ts", tsNanos, ChronoUnit.NANOS) - .at(tsNanos, ChronoUnit.NANOS); - sender.flush(); - } - - assertTableSizeEventually("test_insert_timestamp_nano_units", 1); - assertSqlEventually( - "unit\tts\ttimestamp\n" + - "ns\t2023-09-18T12:01:01.011568000Z\t2023-09-18T12:01:01.011568000Z\n", - "select unit, ts, timestamp from test_insert_timestamp_nano_units"); - } - - @Test - public void testMaxNameLength() throws Exception { - PlainTcpLineChannel channel = new PlainTcpLineChannel(NetworkFacadeImpl.INSTANCE, HOST, getIlpTcpPort(), 1024); - try (AbstractLineTcpSender sender = new LineTcpSenderV2(channel, 1024, 20)) { - try { - sender.table("table_with_long______________________name"); - fail(); - } catch (LineSenderException e) { - assertContains(e.getMessage(), - "table name is too long: [name = table_with_long______________________name, maxNameLength=20]"); - } - - try { - sender.table("tab") - .doubleColumn("column_with_long______________________name", 1.0); - fail(); - } catch (LineSenderException e) { - assertContains(e.getMessage(), - "column name is too long: [name = column_with_long______________________name, maxNameLength=20]"); - } - } - } - - @Test - public void testMultipleVarcharCols() throws Exception { - String table = "test_string_table"; - useTable(table); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - long ts = Micros.floor("2024-02-27"); - sender.table(table) - .stringColumn("string1", "some string") - .stringColumn("string2", "another string") - .stringColumn("string3", "yet another string") - .at(ts, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - "string1\tstring2\tstring3\ttimestamp\n" + - "some string\tanother string\tyet another string\t2024-02-27T00:00:00.000000000Z\n", - "select string1, string2, string3, timestamp from " + table); - } - - @Test - public void testServerIgnoresUnfinishedRows() throws Exception { - String tableName = "test_server_ignores_unfinished_rows"; - useTable(tableName); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - // well-formed row first - sender.table(tableName).longColumn("field0", 42) - .longColumn("field1", 42) - .atNow(); - - // failed validation - sender.table(tableName) - .longColumn("field0", 42) - .longColumn("field1\n", 42); - fail("validation should have failed"); - } catch (LineSenderException e) { - // ignored - } - - // make sure the 2nd unfinished row was not inserted by the server - assertTableSizeEventually(tableName, 1); - } - - @Test - public void testSymbolCapacityReload() throws Exception { - // Tests that the client can send many rows with symbols - String tableName = "test_symbol_capacity_table"; - useTable(tableName); - final int N = 1000; - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - Random rnd = new Random(42); - for (int i = 0; i < N; i++) { - sender.table(tableName) - .symbol("sym1", "sym_" + rnd.nextInt(100)) - .symbol("sym2", "s" + rnd.nextInt(10)) - .doubleColumn("dd", rnd.nextDouble()) - .atNow(); - } - sender.flush(); - } - - assertTableSizeEventually(tableName, N); - } - - @Test - public void testSymbolsCannotBeWrittenAfterBool() throws Exception { - assertSymbolsCannotBeWrittenAfterOtherType(s -> s.boolColumn("columnName", false)); - } - - @Test - public void testSymbolsCannotBeWrittenAfterDouble() throws Exception { - assertSymbolsCannotBeWrittenAfterOtherType(s -> s.doubleColumn("columnName", 42.0)); - } - - @Test - public void testSymbolsCannotBeWrittenAfterLong() throws Exception { - assertSymbolsCannotBeWrittenAfterOtherType(s -> s.longColumn("columnName", 42)); - } - - @Test - public void testSymbolsCannotBeWrittenAfterString() throws Exception { - assertSymbolsCannotBeWrittenAfterOtherType(s -> s.stringColumn("columnName", "42")); - } - - @Test - public void testTimestampIngestV1() throws Exception { - testTimestampIngest("test_timestamp_ingest_v1", "TIMESTAMP", PROTOCOL_VERSION_V1, - "ts\tdts\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834000000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834000000Z\n" + - "2025-11-19T10:55:24.123000000Z\t2025-11-20T10:55:24.834000000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123000000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123000000Z\t2025-11-20T10:55:24.834129000Z\n", - null); - } - - @Test - public void testTimestampIngestV2() throws Exception { - testTimestampIngest("test_timestamp_ingest_v2", "TIMESTAMP", PROTOCOL_VERSION_V2, - "ts\tdts\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834000000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834000000Z\n" + - "2025-11-19T10:55:24.123000000Z\t2025-11-20T10:55:24.834000000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123000000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123000000Z\t2025-11-20T10:55:24.834129000Z\n", - "ts\tdts\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834000000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834000000Z\n" + - "2025-11-19T10:55:24.123000000Z\t2025-11-20T10:55:24.834000000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123000000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123000000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2300-11-19T10:55:24.123456000Z\t2300-11-20T10:55:24.834129000Z\n"); - } - @Test public void testUnfinishedRowDoesNotContainNewLine() { ByteChannel channel = new ByteChannel(); @@ -1359,71 +218,6 @@ public void testUseAfterClose_tsColumn() { assertExceptionOnClosedSender(SET_TABLE_NAME_ACTION, s -> s.timestampColumn("col", 0, ChronoUnit.MICROS)); } - @Test - public void testUseVarcharAsString() throws Exception { - String table = "test_varchar_string_table"; - useTable(table); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - long ts = Micros.floor("2024-02-27"); - String expectedValue = "čćžšđçğéíáýůř"; - sender.table(table) - .stringColumn("string1", expectedValue) - .at(ts, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - "string1\ttimestamp\n" + - "čćžšđçğéíáýůř\t2024-02-27T00:00:00.000000000Z\n", - "select string1, timestamp from " + table); - } - - @Test - public void testWriteAllTypes() throws Exception { - useTable("test_write_all_types"); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - long ts = Micros.floor("2022-02-25"); - sender.table("test_write_all_types") - .longColumn("int_field", 42) - .boolColumn("bool_field", true) - .stringColumn("string_field", "foo") - .doubleColumn("double_field", 42.0) - .timestampColumn("ts_field", ts, ChronoUnit.MICROS) - .at(ts, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually("test_write_all_types", 1); - assertSqlEventually( - "int_field\tbool_field\tstring_field\tdouble_field\tts_field\ttimestamp\n" + - "42\ttrue\tfoo\t42.0\t2022-02-25T00:00:00.000000000Z\t2022-02-25T00:00:00.000000000Z\n", - "select int_field, bool_field, string_field, double_field, ts_field, timestamp from test_write_all_types"); - } - - @Test - public void testWriteLongMinMax() throws Exception { - String table = "test_long_min_max_table"; - useTable(table); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - long ts = Micros.floor("2023-02-22"); - sender.table(table) - .longColumn("max", Long.MAX_VALUE) - .longColumn("min", Long.MIN_VALUE) - .at(ts, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - "max\tmin\ttimestamp\n" + - "9223372036854775807\tnull\t2023-02-22T00:00:00.000000000Z\n", - "select max, min, timestamp from " + table); - } - private static void assertControlCharacterException() { DummyLineChannel channel = new DummyLineChannel(); try (Sender sender = new LineTcpSenderV2(channel, 1000, 127)) { @@ -1437,6 +231,11 @@ private static void assertControlCharacterException() { } } + private static void assertExceptionOnClosedSender() { + assertExceptionOnClosedSender(s -> { + }, LineTcpSenderTest.SET_TABLE_NAME_ACTION); + } + private static void assertExceptionOnClosedSender(Consumer beforeCloseAction, Consumer afterCloseAction) { DummyLineChannel channel = new DummyLineChannel(); @@ -1451,190 +250,12 @@ private static void assertExceptionOnClosedSender(Consumer beforeCloseAc } } - private static void assertExceptionOnClosedSender() { - assertExceptionOnClosedSender(s -> { - }, LineTcpSenderTest.SET_TABLE_NAME_ACTION); - } - private static void assertNoControlCharacter(CharSequence m) { for (int i = 0, n = m.length(); i < n; i++) { assertFalse(Character.isISOControl(m.charAt(i))); } } - private void assertSymbolsCannotBeWrittenAfterOtherType(Consumer otherTypeWriter) throws Exception { - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - sender.table("test_symbols_cannot_be_written_after_other_type"); - otherTypeWriter.accept(sender); - try { - sender.symbol("name", "value"); - fail("symbols cannot be written after any other column type"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "before any other column types"); - sender.atNow(); - } - } - } - - private void testCreateTimestampColumns(long timestamp, ChronoUnit unit, int protocolVersion, - int[] expectedColumnTypes, String expected) throws Exception { - useTable("test_tab1"); - - try (Sender sender = createTcpSender(protocolVersion)) { - long ts_ns = Micros.floor("2025-11-19T10:55:24.123456000Z") * 1000; - long ts_us = Micros.floor("2025-11-19T10:55:24.123456000Z"); - long ts_ms = Micros.floor("2025-11-19T10:55:24.123Z") / 1000; - Instant ts_instant = Instant.ofEpochSecond(ts_ns / 1_000_000_000, ts_ns % 1_000_000_000 + 10); - - if (unit != null) { - sender.table("test_tab1") - .doubleColumn("col1", 1.111) - .timestampColumn("ts_ns", ts_ns, ChronoUnit.NANOS) - .timestampColumn("ts_us", ts_us, ChronoUnit.MICROS) - .timestampColumn("ts_ms", ts_ms, ChronoUnit.MILLIS) - .timestampColumn("ts_instant", ts_instant) - .at(timestamp, unit); - } else { - sender.table("test_tab1") - .doubleColumn("col1", 1.111) - .timestampColumn("ts_ns", ts_ns, ChronoUnit.NANOS) - .timestampColumn("ts_us", ts_us, ChronoUnit.MICROS) - .timestampColumn("ts_ms", ts_ms, ChronoUnit.MILLIS) - .timestampColumn("ts_instant", ts_instant) - .at(Instant.ofEpochSecond(timestamp / 1_000_000_000, timestamp % 1_000_000_000)); - } - - sender.flush(); - } - - assertTableSizeEventually("test_tab1", 1); - assertSqlEventually("column\ttype\n" + - "col1\tDOUBLE\n" + - "timestamp\t" + ColumnType.nameOf(expectedColumnTypes[2]) + "\n" + - "ts_instant\t" + ColumnType.nameOf(expectedColumnTypes[1]) + "\n" + - "ts_ms\tTIMESTAMP\n" + - "ts_ns\t" + ColumnType.nameOf(expectedColumnTypes[0]) + "\n" + - "ts_us\tTIMESTAMP\n", - "select \"column\", \"type\" from table_columns('test_tab1') order by \"column\""); - assertSqlEventually("col1\tts_ns\tts_us\tts_ms\tts_instant\ttimestamp\n" + expected + "\n", - "test_tab1"); - } - - private void testTimestampIngest(String tableName, String timestampType, int protocolVersion, String expected1, - String expected2) throws Exception { - useTable(tableName); - execute("create table " + tableName + " (ts " + timestampType + ", dts " + timestampType - + ") timestamp(dts) partition by DAY BYPASS WAL"); - assertTableExistsEventually(tableName); - - try (Sender sender = createTcpSender(protocolVersion)) { - long ts_ns = Micros.floor("2025-11-19T10:55:24.123456000Z") * 1000; - long dts_ns = Micros.floor("2025-11-20T10:55:24.834129000Z") * 1000; - long ts_us = Micros.floor("2025-11-19T10:55:24.123456000Z"); - long dts_us = Micros.floor("2025-11-20T10:55:24.834129000Z"); - long ts_ms = Micros.floor("2025-11-19T10:55:24.123Z") / 1000; - long dts_ms = Micros.floor("2025-11-20T10:55:24.834Z") / 1000; - Instant tsInstant_ns = Instant.ofEpochSecond(ts_ns / 1_000_000_000, ts_ns % 1_000_000_000 + 10); - Instant dtsInstant_ns = Instant.ofEpochSecond(dts_ns / 1_000_000_000, dts_ns % 1_000_000_000 + 10); - - sender.table(tableName) - .timestampColumn("ts", ts_ns, ChronoUnit.NANOS) - .at(dts_ns, ChronoUnit.NANOS); - sender.table(tableName) - .timestampColumn("ts", ts_us, ChronoUnit.MICROS) - .at(dts_ns, ChronoUnit.NANOS); - sender.table(tableName) - .timestampColumn("ts", ts_ms, ChronoUnit.MILLIS) - .at(dts_ns, ChronoUnit.NANOS); - - sender.table(tableName) - .timestampColumn("ts", ts_ns, ChronoUnit.NANOS) - .at(dts_us, ChronoUnit.MICROS); - sender.table(tableName) - .timestampColumn("ts", ts_us, ChronoUnit.MICROS) - .at(dts_us, ChronoUnit.MICROS); - sender.table(tableName) - .timestampColumn("ts", ts_ms, ChronoUnit.MILLIS) - .at(dts_us, ChronoUnit.MICROS); - - sender.table(tableName) - .timestampColumn("ts", ts_ns, ChronoUnit.NANOS) - .at(dts_ms, ChronoUnit.MILLIS); - sender.table(tableName) - .timestampColumn("ts", ts_us, ChronoUnit.MICROS) - .at(dts_ms, ChronoUnit.MILLIS); - sender.table(tableName) - .timestampColumn("ts", ts_ms, ChronoUnit.MILLIS) - .at(dts_ms, ChronoUnit.MILLIS); - - sender.table(tableName) - .timestampColumn("ts", tsInstant_ns) - .at(dtsInstant_ns); - - sender.flush(); - - assertTableSizeEventually(tableName, 10); - assertSqlEventually(expected1, "select ts, dts from " + tableName); - - try { - // fails for nanos, long overflow - long ts_tooLargeForNanos_us = Micros.floor("2300-11-19T10:55:24.123456000Z"); - long dts_tooLargeForNanos_us = Micros.floor("2300-11-20T10:55:24.834129000Z"); - sender.table(tableName) - .timestampColumn("ts", ts_tooLargeForNanos_us, ChronoUnit.MICROS) - .at(dts_tooLargeForNanos_us, ChronoUnit.MICROS); - sender.flush(); - - if (expected2 == null && protocolVersion == PROTOCOL_VERSION_V1) { - Assert.fail("Exception expected"); - } - } catch (ArithmeticException e) { - if (expected2 == null && protocolVersion == PROTOCOL_VERSION_V1) { - TestUtils.assertContains(e.getMessage(), "long overflow"); - } else { - throw e; - } - } - - assertTableSizeEventually(tableName, expected2 == null ? 10 : 11); - assertSqlEventually(expected2 == null ? expected1 : expected2, "select ts, dts from " + tableName); - } - } - - private void testValueCannotBeInsertedToUuidColumn(String tableName, String value) throws Exception { - useTable(tableName); - execute("CREATE TABLE " + tableName + " (" + - "u1 UUID," + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY NONE BYPASS WAL"); - - assertTableExistsEventually(tableName); - - // this sender fails as the string is not UUID - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - long tsMicros = Micros.floor("2022-02-25"); - sender.table(tableName) - .stringColumn("u1", value) - .at(tsMicros, ChronoUnit.MICROS); - sender.flush(); - } - - // this sender succeeds as the string is in the UUID format - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - long tsMicros = Micros.floor("2022-02-25"); - sender.table(tableName) - .stringColumn("u1", "11111111-1111-1111-1111-111111111111") - .at(tsMicros, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(tableName, 1); - assertSqlEventually( - "u1\tts\n" + - "11111111-1111-1111-1111-111111111111\t2022-02-25T00:00:00.000000000Z\n", - "select u1, ts from " + tableName); - } - private static class DummyLineChannel implements LineChannel { private int closeCounter; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/PlainTcpLineChannelTest.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/PlainTcpLineChannelTest.java index 5f4abff..f1bfc99 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/PlainTcpLineChannelTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/PlainTcpLineChannelTest.java @@ -28,7 +28,7 @@ import io.questdb.client.cutlass.line.tcp.PlainTcpLineChannel; import io.questdb.client.network.NetworkFacade; import io.questdb.client.network.NetworkFacadeImpl; -import io.questdb.client.test.tools.TestUtils; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Test; import static org.junit.Assert.fail; @@ -44,7 +44,7 @@ public int socketTcp(boolean blocking) { @Test public void testConstructorLeak_Hostname_CannotConnect() throws Exception { NetworkFacade nf = NetworkFacadeImpl.INSTANCE; - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try { new PlainTcpLineChannel(nf, "localhost", 1000, 1000); fail("there should be nothing listening on the port 1000, the channel should have failed to connect"); @@ -57,7 +57,7 @@ public void testConstructorLeak_Hostname_CannotConnect() throws Exception { @Test public void testConstructorLeak_Hostname_CannotResolveHost() throws Exception { NetworkFacade nf = NetworkFacadeImpl.INSTANCE; - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try { new PlainTcpLineChannel(nf, "nonsense-fails-to-resolve", 1000, 1000); fail("the host should not resolved and the channel should have failed to connect"); @@ -69,7 +69,7 @@ public void testConstructorLeak_Hostname_CannotResolveHost() throws Exception { @Test public void testConstructorLeak_Hostname_DescriptorsExhausted() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try { new PlainTcpLineChannel(FD_EXHAUSTED_NET_FACADE, "localhost", 1000, 1000); fail("the channel should fail to instantiate when NF fails to create a new socket"); @@ -82,7 +82,7 @@ public void testConstructorLeak_Hostname_DescriptorsExhausted() throws Exception @Test public void testConstructorLeak_IP_CannotConnect() throws Exception { NetworkFacade nf = NetworkFacadeImpl.INSTANCE; - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try { new PlainTcpLineChannel(nf, -1, 1000, 1000); fail("the channel should have failed to connect to address -1"); @@ -94,7 +94,7 @@ public void testConstructorLeak_IP_CannotConnect() throws Exception { @Test public void testConstructorLeak_IP_DescriptorsExhausted() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try { new PlainTcpLineChannel(FD_EXHAUSTED_NET_FACADE, -1, 1000, 1000); fail("the channel should fail to instantiate when NF fails to create a new socket"); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java new file mode 100644 index 0000000..b73a358 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java @@ -0,0 +1,458 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.line.tcp.v4; + +import io.questdb.client.Sender; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import java.time.temporal.ChronoUnit; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +/** + * Test client for ILP allocation profiling. + *

    + * Supports 4 protocol modes: + *

      + *
    • ilp-tcp: Old ILP text protocol over TCP (port 9009)
      • + *
    • ilp-http: Old ILP text protocol over HTTP (port 9000)
      • + *
    • qwp-websocket: New QWP binary protocol over WebSocket (port 9000)
      • + *
    • qwp-udp: New QWP binary protocol over UDP (port 9007)
      • + *
    + *

    + * Sends rows with various column types to exercise all code paths. + * Run with an allocation profiler (async-profiler, JFR, etc.) to find hotspots. + *

    + * Usage: + * 

    + * java -cp ... QwpAllocationTestClient [options]
+ *
+ * Options:
+ *   --protocol=PROTOCOL       Protocol: ilp-tcp, ilp-http, qwp-websocket, qwp-udp (default: qwp-websocket)
+ *   --host=HOST               Server host (default: localhost)
+ *   --port=PORT               Server port (default: 9009 for TCP, 9000 for HTTP/WS, 9007 for UDP)
+ *   --rows=N                  Total rows to send (default: 10000000)
+ *   --batch=N                 Batch/flush size (default: 10000)
+ *   --max-datagram-size=N     Max datagram size in bytes (UDP only, default: 1400)
+ *   --warmup=N                Warmup rows (default: 100000)
+ *   --report=N                Report progress every N rows (default: 1000000)
+ *   --target-throughput=N      Target throughput in rows/sec (0 = unlimited, default: 0)
+ *   --no-warmup               Skip warmup phase
+ *   --help                    Show this help
+ *
+ * Examples:
+ *   QwpAllocationTestClient --protocol=qwp-websocket --rows=1000000 --batch=5000
+ *   QwpAllocationTestClient --protocol=qwp-udp --rows=1000000 --max-datagram-size=8192
+ *   QwpAllocationTestClient --protocol=ilp-tcp --host=remote-server --port=9009
+ *
    + */ +public class QwpAllocationTestClient { + + private static final int DEFAULT_BATCH_SIZE = 10_000; + private static final int DEFAULT_FLUSH_BYTES = 0; // 0 = use protocol default + private static final long DEFAULT_FLUSH_INTERVAL_MS = 0; // 0 = use protocol default + // Default configuration + private static final String DEFAULT_HOST = "localhost"; + private static final int DEFAULT_IN_FLIGHT_WINDOW = 0; // 0 = use protocol default (8) + private static final int DEFAULT_MAX_DATAGRAM_SIZE = 0; // 0 = use protocol default (1400) + private static final int DEFAULT_REPORT_INTERVAL = 1_000_000; + private static final int DEFAULT_ROWS = 80_000_000; + private static final int DEFAULT_TARGET_THROUGHPUT = 0; // 0 = unlimited + private static final int DEFAULT_WARMUP_ROWS = 100_000; + private static final String PROTOCOL_ILP_HTTP = "ilp-http"; + // Protocol modes + private static final String PROTOCOL_ILP_TCP = "ilp-tcp"; + private static final String PROTOCOL_QWP_UDP = "qwp-udp"; + private static final String PROTOCOL_QWP_WEBSOCKET = "qwp-websocket"; + private static final String[] STRINGS = { + "New York", "London", "Tokyo", "Paris", "Berlin", "Sydney", "Toronto", "Singapore", + "Hong Kong", "Dubai", "Mumbai", "Shanghai", "Moscow", "Seoul", "Bangkok", + "Amsterdam", "Zurich", "Frankfurt", "Milan", "Madrid" + }; + // Pre-computed test data to avoid allocation during the test + private static final String[] SYMBOLS = { + "AAPL", "GOOGL", "MSFT", "AMZN", "META", "NVDA", "TSLA", "BRK.A", "JPM", "JNJ", + "V", "PG", "UNH", "HD", "MA", "DIS", "PYPL", "BAC", "ADBE", "CMCSA" + }; + private static final String TABLE_NAME = "ilp_alloc_test"; + + public static void main(String[] args) { + // Parse command-line options + String protocol = PROTOCOL_QWP_WEBSOCKET; + String host = DEFAULT_HOST; + int port = -1; // -1 means use default for protocol + int totalRows = DEFAULT_ROWS; + int batchSize = DEFAULT_BATCH_SIZE; + int flushBytes = DEFAULT_FLUSH_BYTES; + long flushIntervalMs = DEFAULT_FLUSH_INTERVAL_MS; + int inFlightWindow = DEFAULT_IN_FLIGHT_WINDOW; + int maxDatagramSize = DEFAULT_MAX_DATAGRAM_SIZE; + int warmupRows = DEFAULT_WARMUP_ROWS; + int reportInterval = DEFAULT_REPORT_INTERVAL; + int targetThroughput = DEFAULT_TARGET_THROUGHPUT; + boolean isDropTable = true; + + for (String arg : args) { + if (arg.equals("--help") || arg.equals("-h")) { + printUsage(); + System.exit(0); + } else if (arg.startsWith("--protocol=")) { + protocol = arg.substring("--protocol=".length()).toLowerCase(); + } else if (arg.startsWith("--host=")) { + host = arg.substring("--host=".length()); + } else if (arg.startsWith("--port=")) { + port = Integer.parseInt(arg.substring("--port=".length())); + } else if (arg.startsWith("--rows=")) { + totalRows = Integer.parseInt(arg.substring("--rows=".length())); + } else if (arg.startsWith("--batch=")) { + batchSize = Integer.parseInt(arg.substring("--batch=".length())); + } else if (arg.startsWith("--flush-bytes=")) { + flushBytes = Integer.parseInt(arg.substring("--flush-bytes=".length())); + } else if (arg.startsWith("--flush-interval-ms=")) { + flushIntervalMs = Long.parseLong(arg.substring("--flush-interval-ms=".length())); + } else if (arg.startsWith("--in-flight-window=")) { + inFlightWindow = Integer.parseInt(arg.substring("--in-flight-window=".length())); + } else if (arg.startsWith("--max-datagram-size=")) { + maxDatagramSize = Integer.parseInt(arg.substring("--max-datagram-size=".length())); + } else if (arg.startsWith("--warmup=")) { + warmupRows = Integer.parseInt(arg.substring("--warmup=".length())); + } else if (arg.startsWith("--report=")) { + reportInterval = Integer.parseInt(arg.substring("--report=".length())); + } else if (arg.startsWith("--target-throughput=")) { + targetThroughput = Integer.parseInt(arg.substring("--target-throughput=".length())); + } else if (arg.equals("--no-warmup")) { + warmupRows = 0; + } else if (arg.equals("--no-drop")) { + isDropTable = false; + } else if (!arg.startsWith("--")) { + // Legacy positional args: protocol [host] [port] [rows] + protocol = arg.toLowerCase(); + } else { + System.err.println("Unknown option: " + arg); + printUsage(); + System.exit(1); + } + } + + // Use default port if not specified + if (port == -1) { + port = getDefaultPort(protocol); + } + + System.out.println("ILP Allocation Test Client"); + System.out.println("=========================="); + System.out.println("Protocol: " + protocol); + System.out.println("Host: " + host); + System.out.println("Port: " + port); + System.out.println("Total rows: " + String.format("%,d", totalRows)); + System.out.println("Batch size (rows): " + String.format("%,d", batchSize) + (batchSize == 0 ? " (default)" : "")); + System.out.println("Flush bytes: " + (flushBytes == 0 ? "(default)" : String.format("%,d", flushBytes))); + System.out.println("Flush interval: " + (flushIntervalMs == 0 ? "(default)" : flushIntervalMs + " ms")); + System.out.println("In-flight window: " + (inFlightWindow == 0 ? "(default: 8)" : inFlightWindow)); + System.out.println("Max datagram size: " + (maxDatagramSize == 0 ? "(default: 1400)" : maxDatagramSize)); + System.out.println("Warmup rows: " + String.format("%,d", warmupRows)); + System.out.println("Report interval: " + String.format("%,d", reportInterval)); + System.out.println("Target throughput: " + (targetThroughput == 0 ? "(unlimited)" : String.format("%,d", targetThroughput) + " rows/sec")); + System.out.println(); + + try { + if (isDropTable) { + recreateTable(host); + } else { + System.out.println("Skipping table drop (--no-drop)"); + } + runTest(protocol, host, port, totalRows, batchSize, flushBytes, flushIntervalMs, + inFlightWindow, maxDatagramSize, warmupRows, reportInterval, targetThroughput); + } catch (Exception e) { + System.err.println("Error: " + e.getMessage()); + e.printStackTrace(System.err); + System.exit(1); + } + } + + private static Sender createSender( + String protocol, + String host, + int port, + int batchSize, + int flushBytes, + long flushIntervalMs, + int inFlightWindow, + int maxDatagramSize + ) { + switch (protocol) { + case PROTOCOL_ILP_TCP: + return Sender.builder(Sender.Transport.TCP) + .address(host) + .port(port) + .build(); + case PROTOCOL_ILP_HTTP: + return Sender.builder(Sender.Transport.HTTP) + .address(host) + .port(port) + .autoFlushRows(batchSize) + .build(); + case PROTOCOL_QWP_WEBSOCKET: + Sender.LineSenderBuilder wsBuilder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(host) + .port(port); + if (batchSize > 0) wsBuilder.autoFlushRows(batchSize); + if (flushBytes >= 0) wsBuilder.autoFlushBytes(flushBytes); + if (flushIntervalMs > 0) wsBuilder.autoFlushIntervalMillis((int) flushIntervalMs); + if (inFlightWindow > 0) wsBuilder.inFlightWindowSize(inFlightWindow); + return wsBuilder.build(); + case PROTOCOL_QWP_UDP: + Sender.LineSenderBuilder udpBuilder = Sender.builder(Sender.Transport.UDP) + .address(host) + .port(port); + if (maxDatagramSize > 0) udpBuilder.maxDatagramSize(maxDatagramSize); + return udpBuilder.build(); + default: + throw new IllegalArgumentException("Unknown protocol: " + protocol + + ". Use one of: ilp-tcp, ilp-http, qwp-websocket, qwp-udp"); + } + } + + /** + * Estimates the size of a single row in bytes for throughput calculation. + */ + private static int estimatedRowSize() { + // Rough estimate (binary protocol): + // - 2 symbols: ~10 bytes each = 20 bytes + // - 3 longs: 8 bytes each = 24 bytes + // - 4 doubles: 8 bytes each = 32 bytes + // - 1 string: ~10 bytes average + // - 1 boolean: 1 byte + // - 2 timestamps: 8 bytes each = 16 bytes + // - Overhead: ~20 bytes + // Total: ~123 bytes + return 123; + } + + private static int getDefaultPort(String protocol) { + if (PROTOCOL_ILP_HTTP.equals(protocol) || PROTOCOL_QWP_WEBSOCKET.equals(protocol)) { + return 9000; + } + if (PROTOCOL_QWP_UDP.equals(protocol)) { + return 9007; + } + return 9009; + } + + private static void printUsage() { + System.out.println("ILP Allocation Test Client"); + System.out.println(); + System.out.println("Usage: QwpAllocationTestClient [options]"); + System.out.println(); + System.out.println("Options:"); + System.out.println(" --protocol=PROTOCOL Protocol to use (default: qwp-websocket)"); + System.out.println(" --host=HOST Server host (default: localhost)"); + System.out.println(" --port=PORT Server port (default: 9009 for TCP, 9000 for HTTP/WS, 9007 for UDP)"); + System.out.println(" --rows=N Total rows to send (default: 80000000)"); + System.out.println(" --batch=N Auto-flush after N rows (default: 10000)"); + System.out.println(" --flush-bytes=N Auto-flush after N bytes (default: protocol default)"); + System.out.println(" --flush-interval-ms=N Auto-flush after N ms (default: protocol default)"); + System.out.println(" --in-flight-window=N Max batches awaiting server ACK (default: 8, WebSocket only)"); + System.out.println(" --send-queue=N Max batches waiting to send (default: 16, WebSocket only)"); + System.out.println(" --max-datagram-size=N Max datagram size in bytes (default: 1400, UDP only)"); + System.out.println(" --warmup=N Warmup rows (default: 100000)"); + System.out.println(" --report=N Report progress every N rows (default: 1000000)"); + System.out.println(" --target-throughput=N Target throughput in rows/sec (0 = unlimited, default: 0)"); + System.out.println(" --no-warmup Skip warmup phase"); + System.out.println(" --no-drop Don't drop/recreate the table (for parallel clients)"); + System.out.println(" --help Show this help"); + System.out.println(); + System.out.println("Protocols:"); + System.out.println(" ilp-tcp Old ILP text protocol over TCP (default port: 9009)"); + System.out.println(" ilp-http Old ILP text protocol over HTTP (default port: 9000)"); + System.out.println(" qwp-websocket New QWP binary protocol over WebSocket (default port: 9000)"); + System.out.println(" qwp-udp New QWP binary protocol over UDP (default port: 9007)"); + System.out.println(); + System.out.println("Examples:"); + System.out.println(" QwpAllocationTestClient --protocol=qwp-websocket --rows=1000000 --batch=5000"); + System.out.println(" QwpAllocationTestClient --protocol=qwp-udp --rows=1000000 --max-datagram-size=8192"); + System.out.println(" QwpAllocationTestClient --protocol=ilp-tcp --host=remote-server"); + System.out.println(" QwpAllocationTestClient --protocol=ilp-tcp --rows=100000 --no-warmup"); + } + + private static void recreateTable(String host) throws Exception { + Properties properties = new Properties(); + properties.setProperty("user", "admin"); + properties.setProperty("password", "quest"); + properties.setProperty("sslmode", "disable"); + String url = "jdbc:postgresql://" + host + ":8812/qdb"; + try (Connection conn = DriverManager.getConnection(url, properties); + Statement st = conn.createStatement() + ) { + st.execute("DROP TABLE IF EXISTS " + TABLE_NAME); + st.execute("CREATE TABLE " + TABLE_NAME + " (" + + " timestamp TIMESTAMP," + + " exchange SYMBOL," + + " currency SYMBOL," + + " trade_id LONG," + + " volume LONG," + + " price DOUBLE," + + " bid DOUBLE," + + " ask DOUBLE," + + " sequence LONG," + + " spread DOUBLE," + + " venue VARCHAR," + + " is_buy BOOLEAN," + + " event_time TIMESTAMP" + + ") TIMESTAMP(timestamp) PARTITION BY DAY WAL"); + } + System.out.println("Recreated table " + TABLE_NAME); + } + + private static void runTest(String protocol, String host, int port, int totalRows, + int batchSize, int flushBytes, long flushIntervalMs, + int inFlightWindow, int maxDatagramSize, + int warmupRows, int reportInterval, + int targetThroughput) throws IOException { + System.out.println("Connecting to " + host + ":" + port + "..."); + + try (Sender sender = createSender(protocol, host, port, batchSize, flushBytes, flushIntervalMs, + inFlightWindow, maxDatagramSize)) { + System.out.println("Connected! Protocol: " + protocol); + System.out.println(); + + // Warm-up phase + if (warmupRows > 0) { + System.out.println("Warming up (" + String.format("%,d", warmupRows) + " rows)..."); + long warmupStart = System.nanoTime(); + for (int i = 0; i < warmupRows; i++) { + sendRow(sender, i); + } + sender.flush(); + long warmupTime = System.nanoTime() - warmupStart; + System.out.println("Warmup complete in " + TimeUnit.NANOSECONDS.toMillis(warmupTime) + " ms"); + System.out.println(); + + // Give GC a chance to clean up warmup allocations + System.gc(); + Thread.sleep(100); + } + + // Main test phase + System.out.println("Starting main test (" + String.format("%,d", totalRows) + " rows)..."); + if (reportInterval > 0 && reportInterval <= totalRows) { + System.out.println("Progress will be reported every " + String.format("%,d", reportInterval) + " rows"); + } + System.out.println(); + + long startTime = System.nanoTime(); + long lastReportTime = startTime; + int lastReportRows = 0; + // Pacing: check every ~0.1ms worth of rows to keep bursts small + int paceCheckInterval = targetThroughput > 0 ? Math.max(1, targetThroughput / 10_000) : 0; + double nanosPerRow = targetThroughput > 0 ? 1_000_000_000.0 / targetThroughput : 0; + + for (int i = 0; i < totalRows; i++) { + sendRow(sender, i); + + // Pacing: busy-spin until we're back on schedule + if (nanosPerRow > 0 && (i + 1) % paceCheckInterval == 0) { + long expectedElapsedNanos = (long) ((i + 1) * nanosPerRow); + while (System.nanoTime() - startTime < expectedElapsedNanos) { + Thread.onSpinWait(); + } + } + + // Report progress + if (reportInterval > 0 && (i + 1) % reportInterval == 0) { + long now = System.nanoTime(); + long elapsedSinceReport = now - lastReportTime; + int rowsSinceReport = (i + 1) - lastReportRows; + int rowsPerSec = (int) (rowsSinceReport / (elapsedSinceReport / 1_000_000_000.0)); + + System.out.printf("Progress: %,d / %,d rows (%.1f%%) - %,d rows/sec%n", + i + 1, totalRows, + (i + 1) * 100.0 / totalRows, + rowsPerSec); + + lastReportTime = now; + lastReportRows = i + 1; + } + } + + // Final flush + sender.flush(); + + long endTime = System.nanoTime(); + long totalTime = endTime - startTime; + double totalSeconds = totalTime / 1_000_000_000.0; + double rowsPerSecond = totalRows / totalSeconds; + + System.out.println(); + System.out.println("Test Complete!"); + System.out.println("=============="); + System.out.println("Protocol: " + protocol); + System.out.println("Total rows: " + String.format("%,d", totalRows)); + System.out.println("Batch size: " + String.format("%,d", batchSize)); + System.out.println("Total time: " + String.format("%.2f", totalSeconds) + " seconds"); + System.out.println("Throughput: " + String.format("%,.0f", rowsPerSecond) + " rows/second"); + System.out.println("Data rate (before compression): " + String.format("%.2f", ((long) totalRows * estimatedRowSize()) / (1024.0 * 1024.0 * totalSeconds)) + " MB/s (estimated)"); + + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted", e); + } + } + + private static void sendRow(Sender sender, int rowIndex) { + // Base timestamp with small variations + long baseTimestamp = 1704067200000000L; // 2024-01-01 00:00:00 UTC in micros + long timestamp = baseTimestamp + (rowIndex * 1000L) + (rowIndex % 100); + + sender.table(TABLE_NAME) + // Symbol columns + .symbol("exchange", SYMBOLS[rowIndex % SYMBOLS.length]) + .symbol("currency", rowIndex % 2 == 0 ? "USD" : "EUR") + + // Numeric columns + .longColumn("trade_id", rowIndex) + .longColumn("volume", 100 + (rowIndex % 10000)) + .doubleColumn("price", 100.0 + (rowIndex % 1000) * 0.01) + .doubleColumn("bid", 99.5 + (rowIndex % 1000) * 0.01) + .doubleColumn("ask", 100.5 + (rowIndex % 1000) * 0.01) + .longColumn("sequence", rowIndex % 1000000) + .doubleColumn("spread", 0.5 + (rowIndex % 100) * 0.01) + + // String column + .stringColumn("venue", STRINGS[rowIndex % STRINGS.length]) + + // Boolean column + .boolColumn("is_buy", rowIndex % 2 == 0) + + // Additional timestamp column + .timestampColumn("event_time", timestamp - 1000, ChronoUnit.MICROS) + + // Designated timestamp + .at(timestamp, ChronoUnit.MICROS); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java new file mode 100644 index 0000000..bcd4d15 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java @@ -0,0 +1,412 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.line.tcp.v4; + +import io.questdb.client.Sender; + +import java.io.IOException; +import java.time.temporal.ChronoUnit; +import java.util.Random; +import java.util.concurrent.TimeUnit; + +/** + * STAC benchmark ingestion test client. + *

    + * Tests ingestion performance for a STAC-like quotes table with this schema: + * 

    + * CREATE TABLE q (
+ *     s SYMBOL,     -- 4-letter ticker symbol (8512 unique)
+ *     x CHAR,       -- exchange code
+ *     b FLOAT,      -- bid price
+ *     a FLOAT,      -- ask price
+ *     v SHORT,      -- bid volume
+ *     w SHORT,      -- ask volume
+ *     m BOOLEAN,    -- market flag
+ *     T TIMESTAMP   -- designated timestamp
+ * ) timestamp(T) PARTITION BY DAY WAL;
+ *
    + *

    + * The table MUST be pre-created before running this test so the server uses + * the correct narrow column types (FLOAT, SHORT, CHAR). Otherwise ILP + * auto-creation would use DOUBLE, LONG, STRING. + *

    + * Supports 3 protocol modes: + *

      + *
    • ilp-tcp: Old ILP text protocol over TCP (port 9009)
      • + *
    • ilp-http: Old ILP text protocol over HTTP (port 9000)
      • + *
    • qwp-websocket: New QWP binary protocol over WebSocket (port 9000)
      • + *
    + */ +public class StacBenchmarkClient { + + private static final int DEFAULT_BATCH_SIZE = 10_000; + private static final int DEFAULT_FLUSH_BYTES = 0; + private static final long DEFAULT_FLUSH_INTERVAL_MS = 0; + private static final String DEFAULT_HOST = "localhost"; + private static final int DEFAULT_IN_FLIGHT_WINDOW = 0; + private static final int DEFAULT_REPORT_INTERVAL = 1_000_000; + private static final int DEFAULT_ROWS = 80_000_000; + private static final String DEFAULT_TABLE = "q"; + private static final int DEFAULT_WARMUP_ROWS = 100_000; + // Estimated row size for throughput calculation: + // - 1 symbol: ~6 bytes (4-char + overhead) + // - 1 char: 2 bytes + // - 2 floats: 4 bytes each = 8 bytes + // - 2 shorts: 2 bytes each = 4 bytes + // - 1 boolean: 1 byte + // - 1 timestamp: 8 bytes + // - overhead: ~10 bytes + // Total: ~39 bytes + private static final int ESTIMATED_ROW_SIZE = 39; + // Exchange codes (single characters) + private static final char[] EXCHANGES = {'N', 'Q', 'A', 'B', 'C', 'D', 'P', 'Z'}; + // Pre-computed single-char strings to avoid allocation + private static final String[] EXCHANGE_STRINGS = new String[EXCHANGES.length]; + private static final String PROTOCOL_ILP_HTTP = "ilp-http"; + private static final String PROTOCOL_ILP_TCP = "ilp-tcp"; + private static final String PROTOCOL_QWP_WEBSOCKET = "qwp-websocket"; + // 8512 unique 4-letter symbols, as per STAC NYSE benchmark + private static final int SYMBOL_COUNT = 8512; + private static final String[] SYMBOLS = generateSymbols(SYMBOL_COUNT); + // Pre-computed bid base prices per symbol (to generate realistic spreads) + private static final float[] BASE_PRICES = generateBasePrices(SYMBOL_COUNT); + + public static void main(String[] args) { + String protocol = PROTOCOL_QWP_WEBSOCKET; + String host = DEFAULT_HOST; + int port = -1; + int totalRows = DEFAULT_ROWS; + int batchSize = DEFAULT_BATCH_SIZE; + int flushBytes = DEFAULT_FLUSH_BYTES; + long flushIntervalMs = DEFAULT_FLUSH_INTERVAL_MS; + int inFlightWindow = DEFAULT_IN_FLIGHT_WINDOW; + int warmupRows = DEFAULT_WARMUP_ROWS; + int reportInterval = DEFAULT_REPORT_INTERVAL; + String table = DEFAULT_TABLE; + + for (String arg : args) { + if (arg.equals("--help") || arg.equals("-h")) { + printUsage(); + System.exit(0); + } else if (arg.startsWith("--protocol=")) { + protocol = arg.substring("--protocol=".length()).toLowerCase(); + } else if (arg.startsWith("--host=")) { + host = arg.substring("--host=".length()); + } else if (arg.startsWith("--port=")) { + port = Integer.parseInt(arg.substring("--port=".length())); + } else if (arg.startsWith("--rows=")) { + totalRows = Integer.parseInt(arg.substring("--rows=".length())); + } else if (arg.startsWith("--batch=")) { + batchSize = Integer.parseInt(arg.substring("--batch=".length())); + } else if (arg.startsWith("--flush-bytes=")) { + flushBytes = Integer.parseInt(arg.substring("--flush-bytes=".length())); + } else if (arg.startsWith("--flush-interval-ms=")) { + flushIntervalMs = Long.parseLong(arg.substring("--flush-interval-ms=".length())); + } else if (arg.startsWith("--in-flight-window=")) { + inFlightWindow = Integer.parseInt(arg.substring("--in-flight-window=".length())); + } else if (arg.startsWith("--warmup=")) { + warmupRows = Integer.parseInt(arg.substring("--warmup=".length())); + } else if (arg.startsWith("--report=")) { + reportInterval = Integer.parseInt(arg.substring("--report=".length())); + } else if (arg.startsWith("--table=")) { + table = arg.substring("--table=".length()); + } else if (arg.equals("--no-warmup")) { + warmupRows = 0; + } else { + System.err.println("Unknown option: " + arg); + printUsage(); + System.exit(1); + } + } + + if (port == -1) { + port = getDefaultPort(protocol); + } + + System.out.println("STAC Benchmark Ingestion Client"); + System.out.println("================================"); + System.out.println("Protocol: " + protocol); + System.out.println("Host: " + host); + System.out.println("Port: " + port); + System.out.println("Table: " + table); + System.out.println("Total rows: " + String.format("%,d", totalRows)); + System.out.println("Batch size (rows): " + String.format("%,d", batchSize) + (batchSize == 0 ? " (default)" : "")); + System.out.println("Flush bytes: " + (flushBytes == 0 ? "(default)" : String.format("%,d", flushBytes))); + System.out.println("Flush interval: " + (flushIntervalMs == 0 ? "(default)" : flushIntervalMs + " ms")); + System.out.println("In-flight window: " + (inFlightWindow == 0 ? "(default: 8)" : inFlightWindow)); + System.out.println("Warmup rows: " + String.format("%,d", warmupRows)); + System.out.println("Report interval: " + String.format("%,d", reportInterval)); + System.out.println("Symbols: " + String.format("%,d", SYMBOL_COUNT) + " unique 4-letter tickers"); + System.out.println(); + + try { + runTest(protocol, host, port, table, totalRows, batchSize, flushBytes, flushIntervalMs, + inFlightWindow, warmupRows, reportInterval); + } catch (Exception e) { + System.err.println("Error: " + e.getMessage()); + e.printStackTrace(); + System.exit(1); + } + } + + private static Sender createSender(String protocol, String host, int port, + int batchSize, int flushBytes, long flushIntervalMs, + int inFlightWindow) { + switch (protocol) { + case PROTOCOL_ILP_TCP: + return Sender.builder(Sender.Transport.TCP) + .address(host) + .port(port) + .build(); + case PROTOCOL_ILP_HTTP: + return Sender.builder(Sender.Transport.HTTP) + .address(host) + .port(port) + .autoFlushRows(batchSize) + .build(); + case PROTOCOL_QWP_WEBSOCKET: + Sender.LineSenderBuilder b = Sender.builder(Sender.Transport.WEBSOCKET) + .address(host) + .port(port); + if (batchSize > 0) b.autoFlushRows(batchSize); + if (flushBytes > 0) b.autoFlushBytes(flushBytes); + if (flushIntervalMs > 0) b.autoFlushIntervalMillis((int) flushIntervalMs); + if (inFlightWindow > 0) b.inFlightWindowSize(inFlightWindow); + return b.build(); + default: + throw new IllegalArgumentException("Unknown protocol: " + protocol + + ". Use one of: ilp-tcp, ilp-http, qwp-websocket"); + } + } + + /** + * Generates pseudo-random base prices for each symbol. + * Prices range from $1 to $500 to simulate realistic stock prices. + */ + private static float[] generateBasePrices(int count) { + float[] prices = new float[count]; + Random rng = new Random(42); // fixed seed for reproducibility + for (int i = 0; i < count; i++) { + prices[i] = 1.0f + rng.nextFloat() * 499.0f; + } + return prices; + } + + /** + * Generates N unique 4-letter symbols. + * Uses combinations of uppercase letters to produce predictable, reproducible symbols. + */ + private static String[] generateSymbols(int count) { + String[] symbols = new String[count]; + int idx = 0; + // 26^4 = 456,976 possible 4-letter combinations, far more than 8512 + outer: + for (char a = 'A'; a <= 'Z' && idx < count; a++) { + for (char b = 'A'; b <= 'Z' && idx < count; b++) { + for (char c = 'A'; c <= 'Z' && idx < count; c++) { + for (char d = 'A'; d <= 'Z' && idx < count; d++) { + symbols[idx++] = new String(new char[]{a, b, c, d}); + if (idx >= count) break outer; + } + } + } + } + return symbols; + } + + private static int getDefaultPort(String protocol) { + if (PROTOCOL_ILP_HTTP.equals(protocol) || PROTOCOL_QWP_WEBSOCKET.equals(protocol)) { + return 9000; + } + return 9009; + } + + private static void printUsage() { + System.out.println("STAC Benchmark Ingestion Client"); + System.out.println(); + System.out.println("Tests ingestion performance for a STAC-like quotes table."); + System.out.println("The table must be pre-created with the correct schema."); + System.out.println(); + System.out.println("Usage: StacBenchmarkClient [options]"); + System.out.println(); + System.out.println("Options:"); + System.out.println(" --protocol=PROTOCOL Protocol to use (default: qwp-websocket)"); + System.out.println(" --host=HOST Server host (default: localhost)"); + System.out.println(" --port=PORT Server port (default: 9009 for TCP, 9000 for HTTP/WebSocket)"); + System.out.println(" --table=TABLE Table name (default: q)"); + System.out.println(" --rows=N Total rows to send (default: 80000000)"); + System.out.println(" --batch=N Auto-flush after N rows (default: 10000)"); + System.out.println(" --flush-bytes=N Auto-flush after N bytes (default: protocol default)"); + System.out.println(" --flush-interval-ms=N Auto-flush after N ms (default: protocol default)"); + System.out.println(" --in-flight-window=N Max batches awaiting server ACK (default: 8, WebSocket only)"); + System.out.println(" --send-queue=N Max batches waiting to send (default: 16, WebSocket only)"); + System.out.println(" --warmup=N Warmup rows (default: 100000)"); + System.out.println(" --report=N Report progress every N rows (default: 1000000)"); + System.out.println(" --no-warmup Skip warmup phase"); + System.out.println(" --help Show this help"); + System.out.println(); + System.out.println("Protocols:"); + System.out.println(" ilp-tcp Old ILP text protocol over TCP (default port: 9009)"); + System.out.println(" ilp-http Old ILP text protocol over HTTP (default port: 9000)"); + System.out.println(" qwp-websocket New QWP binary protocol over WebSocket (default port: 9000)"); + System.out.println(); + System.out.println("Table schema (must be pre-created):"); + System.out.println(" CREATE TABLE q ("); + System.out.println(" s SYMBOL, x CHAR, b FLOAT, a FLOAT,"); + System.out.println(" v SHORT, w SHORT, m BOOLEAN, T TIMESTAMP"); + System.out.println(" ) timestamp(T) PARTITION BY DAY WAL;"); + } + + private static void runTest(String protocol, String host, int port, String table, + int totalRows, int batchSize, int flushBytes, long flushIntervalMs, + int inFlightWindow, + int warmupRows, int reportInterval) throws IOException { + System.out.println("Connecting to " + host + ":" + port + "..."); + + try (Sender sender = createSender(protocol, host, port, batchSize, flushBytes, flushIntervalMs, + inFlightWindow)) { + System.out.println("Connected! Protocol: " + protocol); + System.out.println(); + + // Warmup phase + if (warmupRows > 0) { + System.out.println("Warming up (" + String.format("%,d", warmupRows) + " rows)..."); + long warmupStart = System.nanoTime(); + for (int i = 0; i < warmupRows; i++) { + sendQuoteRow(sender, table, i); + } + sender.flush(); + long warmupTime = System.nanoTime() - warmupStart; + double warmupRowsPerSec = warmupRows / (warmupTime / 1_000_000_000.0); + System.out.printf("Warmup complete in %d ms (%.0f rows/sec)%n", + TimeUnit.NANOSECONDS.toMillis(warmupTime), warmupRowsPerSec); + System.out.println(); + + System.gc(); + Thread.sleep(100); + } + + // Main test phase + System.out.println("Starting main test (" + String.format("%,d", totalRows) + " rows)..."); + if (reportInterval > 0 && reportInterval <= totalRows) { + System.out.println("Progress will be reported every " + String.format("%,d", reportInterval) + " rows"); + } + System.out.println(); + + long startTime = System.nanoTime(); + long lastReportTime = startTime; + int lastReportRows = 0; + + for (int i = 0; i < totalRows; i++) { + sendQuoteRow(sender, table, i); + + if (reportInterval > 0 && (i + 1) % reportInterval == 0) { + long now = System.nanoTime(); + long elapsedSinceReport = now - lastReportTime; + int rowsSinceReport = (i + 1) - lastReportRows; + double rowsPerSec = rowsSinceReport / (elapsedSinceReport / 1_000_000_000.0); + long totalElapsed = now - startTime; + double overallRowsPerSec = (i + 1) / (totalElapsed / 1_000_000_000.0); + + System.out.printf("Progress: %,d / %,d rows (%.1f%%) - %.0f rows/sec (interval) - %.0f rows/sec (overall)%n", + i + 1, totalRows, + (i + 1) * 100.0 / totalRows, + rowsPerSec, overallRowsPerSec); + + lastReportTime = now; + lastReportRows = i + 1; + } + } + + sender.flush(); + + long endTime = System.nanoTime(); + long totalTime = endTime - startTime; + double totalSeconds = totalTime / 1_000_000_000.0; + double rowsPerSecond = totalRows / totalSeconds; + + System.out.println(); + System.out.println("Test Complete!"); + System.out.println("=============="); + System.out.println("Protocol: " + protocol); + System.out.println("Table: " + table); + System.out.println("Total rows: " + String.format("%,d", totalRows)); + System.out.println("Batch size: " + String.format("%,d", batchSize)); + System.out.println("Total time: " + String.format("%.2f", totalSeconds) + " seconds"); + System.out.println("Throughput: " + String.format("%,.0f", rowsPerSecond) + " rows/second"); + System.out.println("Data rate (before compression): " + String.format("%.2f", + ((long) totalRows * ESTIMATED_ROW_SIZE) / (1024.0 * 1024.0 * totalSeconds)) + " MB/s (estimated)"); + + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted", e); + } + } + + /** + * Sends a single quote row matching the STAC schema. + *

    + * Schema: s SYMBOL, x CHAR, b FLOAT, a FLOAT, v SHORT, w SHORT, m BOOLEAN, T TIMESTAMP + *

    + * The server downcasts doubleColumn->FLOAT, longColumn->SHORT, stringColumn->CHAR + * when the table is pre-created with the correct schema. + */ + private static void sendQuoteRow(Sender sender, String table, int rowIndex) { + int symbolIdx = rowIndex % SYMBOL_COUNT; + int exchangeIdx = rowIndex % EXCHANGES.length; + + // Bid/ask prices: base price with small variation + float basePrice = BASE_PRICES[symbolIdx]; + // Use rowIndex bits for fast pseudo-random variation without Random object + float variation = ((rowIndex * 7 + symbolIdx * 13) % 200 - 100) * 0.01f; + float bid = basePrice + variation; + float ask = bid + 0.01f + (rowIndex % 10) * 0.01f; // spread: 1-10 cents + + // Volumes: 100-32000 range fits SHORT + short bidVol = (short) (100 + ((rowIndex * 3 + symbolIdx) % 31901)); + short askVol = (short) (100 + ((rowIndex * 7 + symbolIdx * 5) % 31901)); + + // Timestamp: 1 day of data with microsecond precision + // 86,400,000,000 micros per day, spread across totalRows + long baseTimestamp = 1704067200000000L; // 2024-01-01 00:00:00 UTC in micros + long timestamp = baseTimestamp + (rowIndex * 10L) + (rowIndex % 7); + + sender.table(table) + .symbol("s", SYMBOLS[symbolIdx]) + .stringColumn("x", EXCHANGE_STRINGS[exchangeIdx]) + .doubleColumn("b", bid) + .doubleColumn("a", ask) + .longColumn("v", bidVol) + .longColumn("w", askVol) + .boolColumn("m", (rowIndex & 1) == 0) + .at(timestamp, ChronoUnit.MICROS); + } + + static { + for (int i = 0; i < EXCHANGES.length; i++) { + EXCHANGE_STRINGS[i] = String.valueOf(EXCHANGES[i]); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/udp/AbstractLineUdpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/line/udp/AbstractLineUdpSenderTest.java deleted file mode 100644 index d5f8e0a..0000000 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/udp/AbstractLineUdpSenderTest.java +++ /dev/null @@ -1,118 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test.cutlass.line.udp; - -import io.questdb.client.cutlass.line.AbstractLineSender; -import io.questdb.client.cutlass.line.LineUdpSender; -import io.questdb.client.std.Numbers; -import io.questdb.client.std.NumericException; -import io.questdb.client.test.cutlass.line.AbstractLineSenderTest; - -/** - * Base class for UDP sender integration tests. - * Provides helper methods for creating UDP senders and managing test tables. - *

    - * Note: UDP is a fire-and-forget protocol, so tests need extra delays - * to account for network latency and server processing time. - */ -public abstract class AbstractLineUdpSenderTest extends AbstractLineSenderTest { - - // Default buffer capacity for UDP sender - protected static final int DEFAULT_BUFFER_CAPACITY = 2048; - - // Default TTL for multicast - protected static final int DEFAULT_TTL = 1; - - /** - * Get localhost IPv4 address as integer. - */ - protected static int getLocalhostIPv4() { - return parseIPv4("127.0.0.1"); - } - - /** - * Parse IPv4 address to integer representation. - */ - protected static int parseIPv4(String address) { - try { - return Numbers.parseIPv4(address); - } catch (NumericException e) { - throw new IllegalArgumentException("Invalid IPv4 address: " + address, e); - } - } - - /** - * Create a UDP sender for multicast. - * - * @param interfaceAddress the interface address to bind to - * @param multicastAddress the multicast group address - * @param bufferCapacity the buffer capacity in bytes - * @param ttl time-to-live for multicast packets - */ - protected AbstractLineSender createMulticastUdpSender( - int interfaceAddress, - int multicastAddress, - int bufferCapacity, - int ttl - ) { - return new LineUdpSender( - interfaceAddress, - multicastAddress, - getIlpUdpPort(), - bufferCapacity, - ttl - ); - } - - /** - * Create a UDP sender with specified buffer size. - * - * @param bufferCapacity the buffer capacity in bytes - */ - protected AbstractLineSender createUdpSender(int bufferCapacity) { - return new LineUdpSender( - getLocalhostIPv4(), // interface address - getTargetIPv4(), // target address - getIlpUdpPort(), // target port - bufferCapacity, - DEFAULT_TTL - ); - } - - /** - * Create a UDP sender with default settings. - * Uses localhost as both interface and target address. - */ - protected AbstractLineSender createUdpSender() { - return createUdpSender(DEFAULT_BUFFER_CAPACITY); - } - - /** - * Get target IPv4 address for UDP sender. - */ - protected int getTargetIPv4() { - return parseIPv4(getQuestDbHost()); - } -} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/udp/LineUdpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/line/udp/LineUdpSenderTest.java deleted file mode 100644 index 66a723c..0000000 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/udp/LineUdpSenderTest.java +++ /dev/null @@ -1,212 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test.cutlass.line.udp; - -import io.questdb.client.cutlass.line.AbstractLineSender; -import org.junit.Test; - -import java.time.Instant; -import java.time.temporal.ChronoUnit; - -/** - * Integration tests for UDP line sender. - *

    - * Note: UDP is a fire-and-forget protocol, so tests need extra delays - * to account for network latency and server processing time. - * These tests require an external QuestDB instance. - */ -public class LineUdpSenderTest extends AbstractLineUdpSenderTest { - - @Test - public void testAllColumnTypes() throws Exception { - String tableName = "test_udp_types"; - useTable(tableName); - try (AbstractLineSender sender = createUdpSender()) { - sender.table(tableName) - .symbol("sym", "abc") - .longColumn("long_col", 42) - .doubleColumn("double_col", 3.14) - .stringColumn("string_col", "hello") - .boolColumn("bool_col", true) - .timestampColumn("ts_col", 1234567890L, ChronoUnit.MICROS) - .atNow(); - sender.flush(); - } - assertTableSizeEventually(tableName, 1); - } - - @Test - public void testCloseAndAssertHelper() throws Exception { - String tableName = "test_udp_close"; - useTable(tableName); - try (AbstractLineSender sender = createUdpSender()) { - sender.table(tableName) - .symbol("device", "dev1") - .longColumn("reading", 100) - .atNow(); - } - assertTableSizeEventually(tableName, 1); - } - - @Test - public void testExplicitTimestamp() throws Exception { - String tableName = "test_udp_ts"; - useTable(tableName); - long ts = Instant.now().toEpochMilli() * 1000; - try (AbstractLineSender sender = createUdpSender()) { - sender.table(tableName) - .symbol("city", "paris") - .longColumn("temp", 15) - .at(ts, ChronoUnit.MICROS); - sender.flush(); - } - assertTableSizeEventually(tableName, 1); - } - - @Test - public void testFlushAndAssertHelper() throws Exception { - String tableName = "udp_helper"; - useTable(tableName); - try (AbstractLineSender sender = createUdpSender()) { - sender.table(tableName) - .symbol("sensor", "s1") - .doubleColumn("value", 123.456) - .atNow(); - flushAndAssertRowCount(sender, tableName, 1); - - sender.table(tableName) - .symbol("sensor", "s2") - .doubleColumn("value", 789.012) - .atNow(); - flushAndAssertRowCount(sender, tableName, 2); - } - } - - @Test - public void testInstantTimestamp() throws Exception { - String tableName = "udp_instant"; - useTable(tableName); - Instant now = Instant.now(); - try (AbstractLineSender sender = createUdpSender()) { - sender.table(tableName) - .symbol("city", "berlin") - .longColumn("temp", 20) - .at(now); - sender.flush(); - } - assertTableSizeEventually(tableName, 1); - } - - @Test - public void testMultipleFlushes() throws Exception { - String tableName = "udp_multiflush"; - useTable(tableName); - try (AbstractLineSender sender = createUdpSender()) { - for (int batch = 0; batch < 5; batch++) { - for (int i = 0; i < 10; i++) { - sender.table(tableName) - .symbol("batch", String.valueOf(batch)) - .longColumn("idx", i) - .atNow(); - } - sender.flush();// Wait between batches - } - } - assertTableSizeEventually(tableName, 50); - } - - @Test - public void testMultipleRows() throws Exception { - String tableName = "udp_multi"; - useTable(tableName); - try (AbstractLineSender sender = createUdpSender()) { - for (int i = 0; i < 10; i++) { - sender.table(tableName) - .symbol("city", "city_" + i) - .longColumn("temp", i * 10) - .atNow(); - } - sender.flush(); - } - assertTableSizeEventually(tableName, 10); - } - - @Test - public void testNullStringValue() throws Exception { - String tableName = "udp_nullstr"; - useTable(tableName); - try (AbstractLineSender sender = createUdpSender()) { - sender.table(tableName) - .symbol("id", "1") - .stringColumn("data", null) - .atNow(); - sender.flush(); - } - assertTableSizeEventually(tableName, 1); - } - - @Test - public void testSimpleInsert() throws Exception { - String tableName = "udp_simple"; - useTable(tableName); - try (AbstractLineSender sender = createUdpSender()) { - sender.table(tableName) - .symbol("city", "london") - .longColumn("temp", 42) - .atNow(); - sender.flush(); - } - assertTableSizeEventually(tableName, 1); - } - - @Test - public void testSpecialCharactersInSymbol() throws Exception { - String tableName = "udp_special"; - useTable(tableName); - try (AbstractLineSender sender = createUdpSender()) { - sender.table(tableName) - .symbol("name", "hello world") - .symbol("path", "/path/to/file") - .longColumn("count", 1) - .atNow(); - sender.flush(); - } - assertTableSizeEventually(tableName, 1); - } - - @Test - public void testUnicodeInString() throws Exception { - String tableName = "udp_unicode"; - useTable(tableName); - try (AbstractLineSender sender = createUdpSender()) { - sender.table(tableName) - .symbol("lang", "ja") - .stringColumn("text", "こんにちは世界") - .atNow(); - sender.flush(); - } - assertTableSizeEventually(tableName, 1); - } -} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/udp/UdpLineChannelTest.java b/core/src/test/java/io/questdb/client/test/cutlass/line/udp/UdpLineChannelTest.java index 1a8289f..426423a 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/udp/UdpLineChannelTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/udp/UdpLineChannelTest.java @@ -28,7 +28,7 @@ import io.questdb.client.cutlass.line.udp.UdpLineChannel; import io.questdb.client.network.NetworkFacade; import io.questdb.client.network.NetworkFacadeImpl; -import io.questdb.client.test.tools.TestUtils; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Test; import static org.junit.Assert.fail; @@ -55,7 +55,7 @@ public int socketUdp() { @Test public void testConstructorLeak_DescriptorsExhausted() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try { new UdpLineChannel(FD_EXHAUSTED_NET_FACADE, 1, 1, 9000, 10); fail("the channel should fail to instantiate when NetworkFacade fails to create a new socket"); @@ -67,7 +67,7 @@ public void testConstructorLeak_DescriptorsExhausted() throws Exception { @Test public void testConstructorLeak_FailsToSendInterface() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try { new UdpLineChannel(FAILS_TO_SET_MULTICAST_IFACE_NET_FACADE, 1, 1, 9000, 10); fail("the channel should fail to instantiate when NF fails to set multicast interface"); @@ -79,7 +79,7 @@ public void testConstructorLeak_FailsToSendInterface() throws Exception { @Test public void testConstructorLeak_FailsToSetTTL() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try { new UdpLineChannel(FAILS_SET_SET_TTL_NET_FACADE, 1, 1, 9000, 10); fail("the channel should fail to instantiate when NF fails to set multicast interface"); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/AsyncModeIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/AsyncModeIntegrationTest.java new file mode 100644 index 0000000..8d95212 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/AsyncModeIntegrationTest.java @@ -0,0 +1,614 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.InFlightWindow; +import io.questdb.client.cutlass.qwp.client.MicrobatchBuffer; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.client.WebSocketSendQueue; +import io.questdb.client.network.PlainSocketFactory; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.*; + +/** + * Integration tests for async mode: double-buffering, send queue, and + * in-flight window working together. + *

    + * These tests verify the interaction between the three async mode components + * ({@link MicrobatchBuffer}, {@link WebSocketSendQueue}, {@link InFlightWindow}) + * without requiring a running QuestDB server. They use {@link FakeWebSocketClient} + * to simulate server behavior and control ACK timing. + */ +public class AsyncModeIntegrationTest { + + /** + * Window of 2. Sends 2 batches (fills window), then enqueues a 3rd to + * occupy the pending slot. The 4th enqueue blocks because the pending + * slot is occupied and the I/O thread cannot poll it (window full). + * Delivering ACKs unblocks the pipeline. + */ + @Test + public void testBackpressureBlocksEnqueueUntilAck() throws Exception { + assertMemoryLeak(() -> { + InFlightWindow window = new InFlightWindow(2, 5_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + AtomicLong highestSent = new AtomicLong(-1); + AtomicLong highestAcked = new AtomicLong(-1); + CountDownLatch twoSent = new CountDownLatch(2); + AtomicBoolean deliverAcks = new AtomicBoolean(false); + + client.setSendBehavior((ptr, len) -> { + highestSent.incrementAndGet(); + twoSent.countDown(); + }); + client.setTryReceiveBehavior(handler -> { + if (deliverAcks.get()) { + long sent = highestSent.get(); + long acked = highestAcked.get(); + if (sent > acked) { + highestAcked.set(sent); + emitAck(handler, sent); + return true; + } + } + return false; + }); + + WebSocketSendQueue queue = null; + MicrobatchBuffer buf0 = new MicrobatchBuffer(256); + MicrobatchBuffer buf1 = new MicrobatchBuffer(256); + + try { + queue = new WebSocketSendQueue(client, window, 3_000, 500); + + // Send 2 batches to fill the window. + buf0.writeByte((byte) 1); + buf0.incrementRowCount(); + buf0.seal(); + queue.enqueue(buf0); + + buf1.writeByte((byte) 2); + buf1.incrementRowCount(); + buf1.seal(); + queue.enqueue(buf1); + + assertTrue("Both batches should be sent", twoSent.await(2, TimeUnit.SECONDS)); + assertEquals("Window should be full", 2, window.getInFlightCount()); + + // Reuse buf0 (recycled by I/O thread) and enqueue a 3rd batch. + // The I/O thread cannot poll it because the window is full. + assertTrue(buf0.awaitRecycled(2, TimeUnit.SECONDS)); + buf0.reset(); + buf0.writeByte((byte) 3); + buf0.incrementRowCount(); + buf0.seal(); + queue.enqueue(buf0); + + // Reuse buf1 and try to enqueue a 4th batch on a background + // thread. It should block because the pending slot is still + // occupied by the 3rd batch. + assertTrue(buf1.awaitRecycled(2, TimeUnit.SECONDS)); + buf1.reset(); + buf1.writeByte((byte) 4); + buf1.incrementRowCount(); + buf1.seal(); + + CountDownLatch enqueueStarted = new CountDownLatch(1); + CountDownLatch enqueueDone = new CountDownLatch(1); + AtomicReference errorRef = new AtomicReference<>(); + WebSocketSendQueue q = queue; + + Thread enqueueThread = new Thread(() -> { + enqueueStarted.countDown(); + try { + q.enqueue(buf1); + } catch (Throwable t) { + errorRef.set(t); + } finally { + enqueueDone.countDown(); + } + }); + enqueueThread.start(); + + assertTrue(enqueueStarted.await(1, TimeUnit.SECONDS)); + Thread.sleep(200); + assertEquals("Enqueue should still be blocked", 1, enqueueDone.getCount()); + + // Deliver ACKs to unblock the pipeline. + deliverAcks.set(true); + + assertTrue("Enqueue should complete after ACK", enqueueDone.await(3, TimeUnit.SECONDS)); + assertNull("No error expected", errorRef.get()); + + queue.flush(); + window.awaitEmpty(); + } finally { + deliverAcks.set(true); + window.acknowledgeUpTo(Long.MAX_VALUE); + closeQuietly(queue); + buf0.close(); + buf1.close(); + client.close(); + } + }); + } + + /** + * Sends 10 batches through 2 alternating buffers with auto-ACK. + * Each buffer cycles through all states multiple times: + * FILLING -> SEALED -> SENDING -> RECYCLED -> FILLING. + */ + @Test + public void testBatchesCycleThroughDoubleBuffers() throws Exception { + assertMemoryLeak(() -> { + InFlightWindow window = new InFlightWindow(4, 5_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + AtomicLong highestSent = new AtomicLong(-1); + AtomicLong highestAcked = new AtomicLong(-1); + + client.setSendBehavior((ptr, len) -> highestSent.incrementAndGet()); + client.setTryReceiveBehavior(handler -> { + long sent = highestSent.get(); + long acked = highestAcked.get(); + if (sent > acked) { + highestAcked.set(sent); + emitAck(handler, sent); + return true; + } + return false; + }); + + WebSocketSendQueue queue = null; + MicrobatchBuffer buf0 = new MicrobatchBuffer(256); + MicrobatchBuffer buf1 = new MicrobatchBuffer(256); + int batchCount = 10; + + try { + queue = new WebSocketSendQueue(client, window, 5_000, 500); + MicrobatchBuffer active = buf0; + + for (int i = 0; i < batchCount; i++) { + if (active.isRecycled()) { + active.reset(); + } + assertTrue("Buffer should be FILLING on iteration " + i, active.isFilling()); + + active.writeByte((byte) (i & 0xFF)); + active.incrementRowCount(); + active.seal(); + queue.enqueue(active); + + // Swap to the other buffer, waiting for it if still in use. + MicrobatchBuffer other = (active == buf0) ? buf1 : buf0; + if (other.isInUse()) { + assertTrue("Other buffer should recycle", + other.awaitRecycled(2, TimeUnit.SECONDS)); + } + if (other.isRecycled()) { + other.reset(); + } + active = other; + } + + queue.flush(); + window.awaitEmpty(); + + assertEquals(batchCount, queue.getTotalBatchesSent()); + assertEquals(0, window.getInFlightCount()); + } finally { + window.acknowledgeUpTo(Long.MAX_VALUE); + closeQuietly(queue); + buf0.close(); + buf1.close(); + client.close(); + } + }); + } + + /** + * The first send blocks in sendBinary (simulating slow I/O). + * The user enqueues a second batch, then tries to swap back to the + * first buffer which is still in SENDING state. The user must wait + * until the I/O thread finishes and recycles the buffer. + */ + @Test + public void testBufferSwapWaitsForSlowSend() throws Exception { + assertMemoryLeak(() -> { + InFlightWindow window = new InFlightWindow(4, 5_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + AtomicLong highestSent = new AtomicLong(-1); + AtomicLong highestAcked = new AtomicLong(-1); + CountDownLatch sendStarted = new CountDownLatch(1); + CountDownLatch sendGate = new CountDownLatch(1); + + client.setSendBehavior((ptr, len) -> { + long seq = highestSent.incrementAndGet(); + if (seq == 0) { + // Block on first send to simulate slow I/O. + sendStarted.countDown(); + try { + if (!sendGate.await(5, TimeUnit.SECONDS)) { + throw new RuntimeException("sendGate timed out"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + }); + client.setTryReceiveBehavior(handler -> { + long sent = highestSent.get(); + long acked = highestAcked.get(); + if (sent > acked) { + highestAcked.set(sent); + emitAck(handler, sent); + return true; + } + return false; + }); + + WebSocketSendQueue queue = null; + MicrobatchBuffer buf0 = new MicrobatchBuffer(256); + MicrobatchBuffer buf1 = new MicrobatchBuffer(256); + + try { + queue = new WebSocketSendQueue(client, window, 5_000, 500); + + // Enqueue buf0. The I/O thread starts sending and blocks. + buf0.writeByte((byte) 1); + buf0.incrementRowCount(); + buf0.seal(); + queue.enqueue(buf0); + + assertTrue("I/O thread should start sending", sendStarted.await(2, TimeUnit.SECONDS)); + assertTrue("buf0 should be in use (SENDING)", buf0.isInUse()); + + // Enqueue buf1 into the pending slot (I/O thread is blocked). + buf1.writeByte((byte) 2); + buf1.incrementRowCount(); + buf1.seal(); + queue.enqueue(buf1); + + // The user wants to reuse buf0, but it is still SENDING. + assertTrue("buf0 should still be in use", buf0.isInUse()); + + // Release the gate so the I/O thread can finish sending buf0. + sendGate.countDown(); + + // buf0 transitions SENDING -> RECYCLED. + assertTrue("buf0 should be recycled after send completes", + buf0.awaitRecycled(2, TimeUnit.SECONDS)); + assertTrue(buf0.isRecycled()); + + // Reset and verify buf0 is reusable. + buf0.reset(); + assertTrue(buf0.isFilling()); + + queue.flush(); + window.awaitEmpty(); + assertEquals(2, queue.getTotalBatchesSent()); + } finally { + sendGate.countDown(); + window.acknowledgeUpTo(Long.MAX_VALUE); + closeQuietly(queue); + buf0.close(); + buf1.close(); + client.close(); + } + }); + } + + /** + * Verifies that {@link WebSocketSendQueue#flush()} returns once the + * batch has been sent over the wire, even though the server has not + * ACKed it yet. The caller must separately call + * {@link InFlightWindow#awaitEmpty()} to wait for the ACK. + */ + @Test + public void testFlushWaitsForSendButNotForAcks() throws Exception { + assertMemoryLeak(() -> { + InFlightWindow window = new InFlightWindow(4, 5_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + AtomicLong highestSent = new AtomicLong(-1); + AtomicBoolean deliverAcks = new AtomicBoolean(false); + + client.setSendBehavior((ptr, len) -> highestSent.incrementAndGet()); + client.setTryReceiveBehavior(handler -> { + if (deliverAcks.get()) { + long sent = highestSent.get(); + if (sent >= 0 && window.getInFlightCount() > 0) { + emitAck(handler, sent); + return true; + } + } + return false; + }); + + WebSocketSendQueue queue = null; + MicrobatchBuffer buf0 = new MicrobatchBuffer(256); + + try { + queue = new WebSocketSendQueue(client, window, 2_000, 500); + + buf0.writeByte((byte) 1); + buf0.incrementRowCount(); + buf0.seal(); + queue.enqueue(buf0); + + // flush() returns once the batch is sent, not when ACKed. + queue.flush(); + assertEquals(1, queue.getTotalBatchesSent()); + assertEquals("Batch should still be in flight", 1, window.getInFlightCount()); + + // Deliver ACK and wait for the window to drain. + deliverAcks.set(true); + window.awaitEmpty(); + assertEquals(0, window.getInFlightCount()); + } finally { + window.acknowledgeUpTo(Long.MAX_VALUE); + closeQuietly(queue); + buf0.close(); + client.close(); + } + }); + } + + /** + * Sends 50 batches through 2 buffers with a window of 4. + * ACKs arrive one-at-a-time (non-cumulative) to test sustained flow + * control under moderate backpressure. + */ + @Test + public void testHighThroughputWithManyBatches() throws Exception { + assertMemoryLeak(() -> { + int batchCount = 50; + int windowSize = 4; + + InFlightWindow window = new InFlightWindow(windowSize, 10_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + AtomicLong highestSent = new AtomicLong(-1); + AtomicLong highestAcked = new AtomicLong(-1); + + client.setSendBehavior((ptr, len) -> highestSent.incrementAndGet()); + client.setTryReceiveBehavior(handler -> { + long sent = highestSent.get(); + long acked = highestAcked.get(); + if (sent > acked) { + // ACK one batch at a time to test sustained flow. + long next = acked + 1; + highestAcked.set(next); + emitAck(handler, next); + return true; + } + return false; + }); + + WebSocketSendQueue queue = null; + MicrobatchBuffer buf0 = new MicrobatchBuffer(256); + MicrobatchBuffer buf1 = new MicrobatchBuffer(256); + + try { + queue = new WebSocketSendQueue(client, window, 10_000, 2_000); + MicrobatchBuffer active = buf0; + + for (int i = 0; i < batchCount; i++) { + if (!active.isFilling()) { + if (active.isInUse()) { + assertTrue("Buffer should recycle on iteration " + i, + active.awaitRecycled(5, TimeUnit.SECONDS)); + } + active.reset(); + } + + active.writeByte((byte) (i & 0xFF)); + active.incrementRowCount(); + active.seal(); + queue.enqueue(active); + + active = (active == buf0) ? buf1 : buf0; + } + + queue.flush(); + window.awaitEmpty(); + + assertEquals(batchCount, queue.getTotalBatchesSent()); + assertEquals(0, window.getInFlightCount()); + } finally { + window.acknowledgeUpTo(Long.MAX_VALUE); + closeQuietly(queue); + buf0.close(); + buf1.close(); + client.close(); + } + }); + } + + /** + * The server ACKs the first batch but returns a WRITE_ERROR for the + * second. {@link WebSocketSendQueue#flush()} completes (both batches + * were sent) but {@link InFlightWindow#awaitEmpty()} surfaces the error. + */ + @Test + public void testServerErrorPropagatesOnFlush() throws Exception { + assertMemoryLeak(() -> { + InFlightWindow window = new InFlightWindow(4, 5_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + AtomicLong highestSent = new AtomicLong(-1); + AtomicLong highestDelivered = new AtomicLong(-1); + + client.setSendBehavior((ptr, len) -> highestSent.incrementAndGet()); + client.setTryReceiveBehavior(handler -> { + long sent = highestSent.get(); + long delivered = highestDelivered.get(); + if (sent > delivered) { + long next = delivered + 1; + highestDelivered.set(next); + if (next == 1) { + emitError(handler, next, WebSocketResponse.STATUS_WRITE_ERROR, "disk full"); + } else { + emitAck(handler, next); + } + return true; + } + return false; + }); + + WebSocketSendQueue queue = null; + MicrobatchBuffer buf0 = new MicrobatchBuffer(256); + MicrobatchBuffer buf1 = new MicrobatchBuffer(256); + + try { + queue = new WebSocketSendQueue(client, window, 2_000, 500); + + buf0.writeByte((byte) 1); + buf0.incrementRowCount(); + buf0.seal(); + queue.enqueue(buf0); + + buf1.writeByte((byte) 2); + buf1.incrementRowCount(); + buf1.seal(); + queue.enqueue(buf1); + + // flush() waits for the queue to drain (both batches sent). + queue.flush(); + + // awaitEmpty() surfaces the server error for batch 1. + try { + window.awaitEmpty(); + fail("Expected server error to propagate"); + } catch (LineSenderException e) { + assertTrue("Error should mention server failure", + e.getMessage().contains("disk full") || e.getMessage().contains("Server error")); + } + } finally { + closeQuietly(queue); + buf0.close(); + buf1.close(); + client.close(); + } + }); + } + + private static void closeQuietly(WebSocketSendQueue queue) { + if (queue != null) { + queue.close(); + } + } + + private static void emitAck(WebSocketFrameHandler handler, long sequence) { + WebSocketResponse resp = WebSocketResponse.success(sequence); + int size = resp.serializedSize(); + long ptr = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); + try { + resp.writeTo(ptr); + handler.onBinaryMessage(ptr, size); + } finally { + Unsafe.free(ptr, size, MemoryTag.NATIVE_DEFAULT); + } + } + + private static void emitError(WebSocketFrameHandler handler, long sequence, byte status, String message) { + WebSocketResponse resp = WebSocketResponse.error(sequence, status, message); + int size = resp.serializedSize(); + long ptr = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); + try { + resp.writeTo(ptr); + handler.onBinaryMessage(ptr, size); + } finally { + Unsafe.free(ptr, size, MemoryTag.NATIVE_DEFAULT); + } + } + + private interface SendBehavior { + void send(long dataPtr, int length); + } + + private interface TryReceiveBehavior { + boolean tryReceive(WebSocketFrameHandler handler); + } + + private static class FakeWebSocketClient extends WebSocketClient { + private volatile boolean connected = true; + private volatile SendBehavior sendBehavior = (dataPtr, length) -> {}; + private volatile TryReceiveBehavior tryReceiveBehavior = handler -> false; + + private FakeWebSocketClient() { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + } + + @Override + public void close() { + connected = false; + super.close(); + } + + @Override + public boolean isConnected() { + return connected; + } + + @Override + public void sendBinary(long dataPtr, int length) { + sendBehavior.send(dataPtr, length); + } + + public void setSendBehavior(SendBehavior sendBehavior) { + this.sendBehavior = sendBehavior; + } + + public void setTryReceiveBehavior(TryReceiveBehavior tryReceiveBehavior) { + this.tryReceiveBehavior = tryReceiveBehavior; + } + + @Override + public boolean tryReceiveFrame(WebSocketFrameHandler handler) { + return tryReceiveBehavior.tryReceive(handler); + } + + @Override + protected void ioWait(int timeout, int op) { + // no-op + } + + @Override + protected void setupIoWait() { + // no-op + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java new file mode 100644 index 0000000..7dbd212 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java @@ -0,0 +1,647 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; +import io.questdb.client.cutlass.qwp.client.QwpBufferWriter; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.ObjList; +import io.questdb.client.std.Unsafe; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; +import org.junit.Assert; +import org.junit.Test; + +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; + +/** + * Comprehensive tests for delta symbol dictionary encoding and decoding. + *

    + * Tests cover: + * - Multiple tables sharing the same global dictionary + * - Multiple batches with progressive symbol accumulation + * - Reconnection scenarios where the dictionary resets + * - Multiple symbol columns in the same table + * - Edge cases (empty batches, no symbols, etc.) + */ +public class DeltaSymbolDictionaryTest { + + @Test + public void testEdgeCase_batchWithNoSymbols() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Table with only non-symbol columns + try (QwpTableBuffer batch = new QwpTableBuffer("metrics")) { + QwpTableBuffer.ColumnBuffer valueCol = batch.getOrCreateColumn("value", TYPE_LONG, false); + valueCol.addLong(100L); + batch.nextRow(); + + // MaxId is -1 (no symbols) + int batchMaxId = -1; + + // Can still encode with delta dict (empty delta) + int size = encoder.encodeWithDeltaDict(batch, globalDict, -1, batchMaxId, false); + Assert.assertTrue(size > 0); + + // Verify flag is set + QwpBufferWriter buf = encoder.getBuffer(); + byte flags = Unsafe.getUnsafe().getByte(buf.getBufferPtr() + HEADER_OFFSET_FLAGS); + Assert.assertTrue((flags & FLAG_DELTA_SYMBOL_DICT) != 0); + } + } + }); + } + + @Test + public void testEdgeCase_duplicateSymbolsInBatch() throws Exception { + assertMemoryLeak(() -> { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + try (QwpTableBuffer batch = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = batch.getOrCreateColumn("sym", TYPE_SYMBOL, false); + + // Same symbol used multiple times + int aaplId = globalDict.getOrAddSymbol("AAPL"); + col.addSymbolWithGlobalId("AAPL", aaplId); + batch.nextRow(); + col.addSymbolWithGlobalId("AAPL", aaplId); + batch.nextRow(); + col.addSymbolWithGlobalId("AAPL", aaplId); + batch.nextRow(); + + Assert.assertEquals(3, batch.getRowCount()); + Assert.assertEquals(1, globalDict.size()); // Only 1 unique symbol + + int maxGlobalId = col.getMaxGlobalSymbolId(); + Assert.assertEquals(0, maxGlobalId); // Max ID is 0 (AAPL) + } + }); + } + + @Test + public void testEdgeCase_emptyBatch() throws Exception { + assertMemoryLeak(() -> { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Pre-populate dictionary and send + globalDict.getOrAddSymbol("AAPL"); + int maxSentSymbolId = 0; + + // Empty batch (no rows, no symbols used) + try (QwpTableBuffer emptyBatch = new QwpTableBuffer("test")) { + Assert.assertEquals(0, emptyBatch.getRowCount()); + + // Delta should still work (deltaCount = 0) + int deltaStart = maxSentSymbolId + 1; + int deltaCount = 0; + Assert.assertEquals(1, deltaStart); + Assert.assertEquals(0, deltaCount); + } + }); + } + + @Test + public void testEdgeCase_gapFill() throws Exception { + assertMemoryLeak(() -> { + // Client dictionary: AAPL(0), GOOG(1), MSFT(2), TSLA(3) + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + globalDict.getOrAddSymbol("AAPL"); + globalDict.getOrAddSymbol("GOOG"); + globalDict.getOrAddSymbol("MSFT"); + globalDict.getOrAddSymbol("TSLA"); + + // Batch uses AAPL(0) and TSLA(3), skipping GOOG(1) and MSFT(2) + // Delta must include gap-fill: send all symbols from maxSentSymbolId+1 to batchMaxId + int maxSentSymbolId = -1; + int batchMaxId = 3; // TSLA + + int deltaStart = maxSentSymbolId + 1; + int deltaCount = batchMaxId - maxSentSymbolId; + + // Must send symbols 0, 1, 2, 3 (even though 1, 2 aren't used in this batch) + Assert.assertEquals(0, deltaStart); + Assert.assertEquals(4, deltaCount); + + // This ensures server has contiguous dictionary + for (int id = deltaStart; id < deltaStart + deltaCount; id++) { + String symbol = globalDict.getSymbol(id); + Assert.assertNotNull("Symbol " + id + " should exist", symbol); + } + }); + } + + @Test + public void testEdgeCase_largeSymbolDictionary() throws Exception { + assertMemoryLeak(() -> { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Add 1000 unique symbols + for (int i = 0; i < 1000; i++) { + int id = globalDict.getOrAddSymbol("SYM_" + i); + Assert.assertEquals(i, id); + } + + Assert.assertEquals(1000, globalDict.size()); + + // Send first batch with symbols 0-99 + int maxSentSymbolId = 99; + + // Next batch uses symbols 0-199, delta is 100-199 + int deltaStart = maxSentSymbolId + 1; + int deltaCount = 199 - maxSentSymbolId; + Assert.assertEquals(100, deltaStart); + Assert.assertEquals(100, deltaCount); + }); + } + + @Test + public void testEdgeCase_nullSymbolValues() throws Exception { + assertMemoryLeak(() -> { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + try (QwpTableBuffer batch = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = batch.getOrCreateColumn("sym", TYPE_SYMBOL, true); // useNullBitmap + + int aaplId = globalDict.getOrAddSymbol("AAPL"); + col.addSymbolWithGlobalId("AAPL", aaplId); + batch.nextRow(); + + col.addSymbol(null); // NULL value + batch.nextRow(); + + col.addSymbolWithGlobalId("AAPL", aaplId); + batch.nextRow(); + + Assert.assertEquals(3, batch.getRowCount()); + // Dictionary only has 1 symbol (AAPL), NULL doesn't add to dictionary + Assert.assertEquals(1, globalDict.size()); + } + }); + } + + @Test + public void testEdgeCase_unicodeSymbols() throws Exception { + assertMemoryLeak(() -> { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Unicode symbols + int id1 = globalDict.getOrAddSymbol("日本"); + int id2 = globalDict.getOrAddSymbol("中国"); + int id3 = globalDict.getOrAddSymbol("한국"); + int id4 = globalDict.getOrAddSymbol("Émoji🚀"); + + Assert.assertEquals(0, id1); + Assert.assertEquals(1, id2); + Assert.assertEquals(2, id3); + Assert.assertEquals(3, id4); + + Assert.assertEquals("日本", globalDict.getSymbol(0)); + Assert.assertEquals("中国", globalDict.getSymbol(1)); + Assert.assertEquals("한국", globalDict.getSymbol(2)); + Assert.assertEquals("Émoji🚀", globalDict.getSymbol(3)); + }); + } + + @Test + public void testEdgeCase_veryLongSymbol() throws Exception { + assertMemoryLeak(() -> { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Create a very long symbol (1000 chars) + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 1000; i++) { + sb.append('X'); + } + String longSymbol = sb.toString(); + + int id = globalDict.getOrAddSymbol(longSymbol); + Assert.assertEquals(0, id); + + String retrieved = globalDict.getSymbol(0); + Assert.assertEquals(longSymbol, retrieved); + Assert.assertEquals(1000, retrieved.length()); + }); + } + + @Test + public void testMultipleBatches_encodeAndDecode() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + GlobalSymbolDictionary clientDict = new GlobalSymbolDictionary(); + ObjList serverDict = new ObjList<>(); + int maxSentSymbolId = -1; + + int aaplId = clientDict.getOrAddSymbol("AAPL"); + int googId = clientDict.getOrAddSymbol("GOOG"); + + try (QwpTableBuffer batch1 = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col1 = batch1.getOrCreateColumn("sym", TYPE_SYMBOL, false); + + col1.addSymbolWithGlobalId("AAPL", aaplId); + batch1.nextRow(); + col1.addSymbolWithGlobalId("GOOG", googId); + batch1.nextRow(); + + int batch1MaxId = 1; + int size1 = encoder.encodeWithDeltaDict(batch1, clientDict, maxSentSymbolId, batch1MaxId, false); + Assert.assertTrue(size1 > 0); + maxSentSymbolId = batch1MaxId; + + // Decode on server side + QwpBufferWriter buf1 = encoder.getBuffer(); + decodeAndAccumulateDict(buf1.getBufferPtr(), size1, serverDict); + + // Verify server dictionary + Assert.assertEquals(2, serverDict.size()); + Assert.assertEquals("AAPL", serverDict.get(0)); + Assert.assertEquals("GOOG", serverDict.get(1)); + } + + try (QwpTableBuffer batch2 = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col2 = batch2.getOrCreateColumn("sym", TYPE_SYMBOL, false); + + int msftId = clientDict.getOrAddSymbol("MSFT"); + col2.addSymbolWithGlobalId("AAPL", aaplId); // Existing + batch2.nextRow(); + col2.addSymbolWithGlobalId("MSFT", msftId); // New + batch2.nextRow(); + + int batch2MaxId = 2; + int size2 = encoder.encodeWithDeltaDict(batch2, clientDict, maxSentSymbolId, batch2MaxId, false); + Assert.assertTrue(size2 > 0); + maxSentSymbolId = batch2MaxId; + + // Decode batch 2 + QwpBufferWriter buf2 = encoder.getBuffer(); + decodeAndAccumulateDict(buf2.getBufferPtr(), size2, serverDict); + + // Server dictionary should now have 3 symbols + Assert.assertEquals(3, serverDict.size()); + Assert.assertEquals("MSFT", serverDict.get(2)); + } + } + }); + } + + @Test + public void testMultipleBatches_progressiveSymbolAccumulation() throws Exception { + assertMemoryLeak(() -> { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Batch 1: AAPL, GOOG + int aaplId = globalDict.getOrAddSymbol("AAPL"); + int googId = globalDict.getOrAddSymbol("GOOG"); + int batch1MaxId = Math.max(aaplId, googId); + + // Simulate sending batch 1 - maxSentSymbolId = 1 after send + int maxSentSymbolId = batch1MaxId; // 1 + + // Batch 2: AAPL (existing), MSFT (new), TSLA (new) + globalDict.getOrAddSymbol("AAPL"); // Returns 0, already exists + int msftId = globalDict.getOrAddSymbol("MSFT"); + int tslaId = globalDict.getOrAddSymbol("TSLA"); + int batch2MaxId = Math.max(msftId, tslaId); + + // Delta for batch 2 should be [2, 3] (MSFT, TSLA) + int deltaStart = maxSentSymbolId + 1; + int deltaCount = batch2MaxId - maxSentSymbolId; + Assert.assertEquals(2, deltaStart); + Assert.assertEquals(2, deltaCount); + + // Simulate sending batch 2 + maxSentSymbolId = batch2MaxId; // 3 + + // Batch 3: All existing symbols (no delta needed) + globalDict.getOrAddSymbol("AAPL"); + globalDict.getOrAddSymbol("GOOG"); + int batch3MaxId = 1; // Max used is GOOG(1) + + deltaStart = maxSentSymbolId + 1; + deltaCount = Math.max(0, batch3MaxId - maxSentSymbolId); + Assert.assertEquals(4, deltaStart); + Assert.assertEquals(0, deltaCount); // No new symbols + }); + } + + @Test + public void testMultipleTables_encodedInSameBatch() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Create two tables + try (QwpTableBuffer table1 = new QwpTableBuffer("trades"); + QwpTableBuffer table2 = new QwpTableBuffer("quotes")) { + + // Table 1: ticker column + QwpTableBuffer.ColumnBuffer col1 = table1.getOrCreateColumn("ticker", TYPE_SYMBOL, false); + int aaplId = globalDict.getOrAddSymbol("AAPL"); + int googId = globalDict.getOrAddSymbol("GOOG"); + col1.addSymbolWithGlobalId("AAPL", aaplId); + table1.nextRow(); + col1.addSymbolWithGlobalId("GOOG", googId); + table1.nextRow(); + + // Table 2: symbol column (different name, but shares dictionary) + QwpTableBuffer.ColumnBuffer col2 = table2.getOrCreateColumn("symbol", TYPE_SYMBOL, false); + int msftId = globalDict.getOrAddSymbol("MSFT"); + col2.addSymbolWithGlobalId("AAPL", aaplId); // Reuse AAPL + table2.nextRow(); + col2.addSymbolWithGlobalId("MSFT", msftId); + table2.nextRow(); + + // Encode first table with delta dict + int confirmedMaxId = -1; + int batchMaxId = 2; // AAPL(0), GOOG(1), MSFT(2) + + int size = encoder.encodeWithDeltaDict(table1, globalDict, confirmedMaxId, batchMaxId, false); + Assert.assertTrue(size > 0); + + // Verify delta section contains all 3 symbols + QwpBufferWriter buf = encoder.getBuffer(); + long ptr = buf.getBufferPtr(); + + byte flags = Unsafe.getUnsafe().getByte(ptr + HEADER_OFFSET_FLAGS); + Assert.assertTrue((flags & FLAG_DELTA_SYMBOL_DICT) != 0); + + // After header: deltaStart=0, deltaCount=3 + long pos = ptr + HEADER_SIZE; + int deltaStart = readVarint(pos); + Assert.assertEquals(0, deltaStart); + } + } + }); + } + + @Test + public void testMultipleTables_multipleSymbolColumns() throws Exception { + assertMemoryLeak(() -> { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + try (QwpTableBuffer table = new QwpTableBuffer("market_data")) { + + // Column 1: exchange + QwpTableBuffer.ColumnBuffer exchangeCol = table.getOrCreateColumn("exchange", TYPE_SYMBOL, false); + int nyseId = globalDict.getOrAddSymbol("NYSE"); + int nasdaqId = globalDict.getOrAddSymbol("NASDAQ"); + + // Column 2: currency + QwpTableBuffer.ColumnBuffer currencyCol = table.getOrCreateColumn("currency", TYPE_SYMBOL, false); + int usdId = globalDict.getOrAddSymbol("USD"); + int eurId = globalDict.getOrAddSymbol("EUR"); + + // Column 3: ticker + QwpTableBuffer.ColumnBuffer tickerCol = table.getOrCreateColumn("ticker", TYPE_SYMBOL, false); + int aaplId = globalDict.getOrAddSymbol("AAPL"); + + // Add row with all three columns + exchangeCol.addSymbolWithGlobalId("NYSE", nyseId); + currencyCol.addSymbolWithGlobalId("USD", usdId); + tickerCol.addSymbolWithGlobalId("AAPL", aaplId); + table.nextRow(); + + exchangeCol.addSymbolWithGlobalId("NASDAQ", nasdaqId); + currencyCol.addSymbolWithGlobalId("EUR", eurId); + tickerCol.addSymbolWithGlobalId("AAPL", aaplId); // Reuse AAPL + table.nextRow(); + + // All symbols share the same global dictionary + Assert.assertEquals(5, globalDict.size()); + Assert.assertEquals("NYSE", globalDict.getSymbol(0)); + Assert.assertEquals("NASDAQ", globalDict.getSymbol(1)); + Assert.assertEquals("USD", globalDict.getSymbol(2)); + Assert.assertEquals("EUR", globalDict.getSymbol(3)); + Assert.assertEquals("AAPL", globalDict.getSymbol(4)); + } + }); + } + + @Test + public void testMultipleTables_sharedGlobalDictionary() throws Exception { + assertMemoryLeak(() -> { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Table 1 uses symbols AAPL, GOOG + int aaplId = globalDict.getOrAddSymbol("AAPL"); + int googId = globalDict.getOrAddSymbol("GOOG"); + + // Table 2 uses symbols AAPL (reused), MSFT (new) + int aaplId2 = globalDict.getOrAddSymbol("AAPL"); // Should return same ID + int msftId = globalDict.getOrAddSymbol("MSFT"); + + // Verify deduplication + Assert.assertEquals(0, aaplId); + Assert.assertEquals(1, googId); + Assert.assertEquals(0, aaplId2); // Same as aaplId + Assert.assertEquals(2, msftId); + + // Total symbols should be 3 + Assert.assertEquals(3, globalDict.size()); + }); + } + + @Test + public void testReconnection_fullDeltaAfterReconnect() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + GlobalSymbolDictionary clientDict = new GlobalSymbolDictionary(); + + // First connection: add symbols + int aaplId = clientDict.getOrAddSymbol("AAPL"); + clientDict.getOrAddSymbol("GOOG"); + + // Send batch - maxSentSymbolId = 1 + int maxSentSymbolId = 1; + + // Reconnect - reset maxSentSymbolId + maxSentSymbolId = -1; + + // Create new batch using existing symbols + try (QwpTableBuffer batch = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = batch.getOrCreateColumn("sym", TYPE_SYMBOL, false); + col.addSymbolWithGlobalId("AAPL", aaplId); + batch.nextRow(); + + // Encode - should send full delta (all symbols from 0) + int size = encoder.encodeWithDeltaDict(batch, clientDict, maxSentSymbolId, 1, false); + Assert.assertTrue(size > 0); + + // Verify deltaStart is 0 + QwpBufferWriter buf = encoder.getBuffer(); + long pos = buf.getBufferPtr() + HEADER_SIZE; + int deltaStart = readVarint(pos); + Assert.assertEquals(0, deltaStart); + } + } + }); + } + + @Test + public void testReconnection_resetsWatermark() throws Exception { + assertMemoryLeak(() -> { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Build up dictionary and "send" some symbols + globalDict.getOrAddSymbol("AAPL"); + globalDict.getOrAddSymbol("GOOG"); + globalDict.getOrAddSymbol("MSFT"); + + int maxSentSymbolId = 2; + + // Simulate reconnection - reset maxSentSymbolId + maxSentSymbolId = -1; + Assert.assertEquals(-1, maxSentSymbolId); + + // Global dictionary is NOT cleared (it's client-side) + Assert.assertEquals(3, globalDict.size()); + + // Next batch must send full delta from 0 + int deltaStart = maxSentSymbolId + 1; + Assert.assertEquals(0, deltaStart); + }); + } + + @Test + public void testReconnection_serverDictionaryCleared() throws Exception { + assertMemoryLeak(() -> { + ObjList serverDict = new ObjList<>(); + + // Simulate first connection + serverDict.add("AAPL"); + serverDict.add("GOOG"); + Assert.assertEquals(2, serverDict.size()); + + // Simulate reconnection - server clears dictionary + serverDict.clear(); + Assert.assertEquals(0, serverDict.size()); + + // New connection starts fresh + serverDict.add("MSFT"); + Assert.assertEquals(1, serverDict.size()); + Assert.assertEquals("MSFT", serverDict.get(0)); + }); + } + + @Test + public void testServerSide_accumulateDelta() throws Exception { + assertMemoryLeak(() -> { + ObjList serverDict = new ObjList<>(); + + // First batch: symbols 0-2 + accumulateDelta(serverDict, 0, new String[]{"AAPL", "GOOG", "MSFT"}); + + Assert.assertEquals(3, serverDict.size()); + Assert.assertEquals("AAPL", serverDict.get(0)); + Assert.assertEquals("GOOG", serverDict.get(1)); + Assert.assertEquals("MSFT", serverDict.get(2)); + + // Second batch: symbols 3-4 + accumulateDelta(serverDict, 3, new String[]{"TSLA", "AMZN"}); + + Assert.assertEquals(5, serverDict.size()); + Assert.assertEquals("TSLA", serverDict.get(3)); + Assert.assertEquals("AMZN", serverDict.get(4)); + + // Third batch: no new symbols (empty delta) + accumulateDelta(serverDict, 5, new String[]{}); + Assert.assertEquals(5, serverDict.size()); + }); + } + + @Test + public void testServerSide_resolveSymbol() throws Exception { + assertMemoryLeak(() -> { + ObjList serverDict = new ObjList<>(); + serverDict.add("AAPL"); + serverDict.add("GOOG"); + serverDict.add("MSFT"); + + // Resolve by global ID + Assert.assertEquals("AAPL", serverDict.get(0)); + Assert.assertEquals("GOOG", serverDict.get(1)); + Assert.assertEquals("MSFT", serverDict.get(2)); + }); + } + + private void accumulateDelta(ObjList serverDict, int deltaStart, String[] symbols) { + // Ensure capacity + while (serverDict.size() < deltaStart + symbols.length) { + serverDict.add(null); + } + // Add symbols + for (int i = 0; i < symbols.length; i++) { + serverDict.setQuick(deltaStart + i, symbols[i]); + } + } + + private void decodeAndAccumulateDict(long ptr, int size, ObjList serverDict) { + // Parse header + byte flags = Unsafe.getUnsafe().getByte(ptr + HEADER_OFFSET_FLAGS); + if ((flags & FLAG_DELTA_SYMBOL_DICT) == 0) { + return; // No delta dict + } + + // Parse delta section after header + long pos = ptr + HEADER_SIZE; + + // Read deltaStart + int deltaStart = readVarint(pos); + pos += 1; // Assuming single-byte varint + + // Read deltaCount + int deltaCount = readVarint(pos); + pos += 1; + + // Ensure capacity + while (serverDict.size() < deltaStart + deltaCount) { + serverDict.add(null); + } + + // Read symbols + for (int i = 0; i < deltaCount; i++) { + int len = readVarint(pos); + pos += 1; + + byte[] bytes = new byte[len]; + for (int j = 0; j < len; j++) { + bytes[j] = Unsafe.getUnsafe().getByte(pos + j); + } + pos += len; + + serverDict.setQuick(deltaStart + i, new String(bytes, java.nio.charset.StandardCharsets.UTF_8)); + } + } + + private int readVarint(long address) { + byte b = Unsafe.getUnsafe().getByte(address); + if ((b & 0x80) == 0) { + return b & 0x7F; + } + // For simplicity, only handle single-byte varints in tests + return b & 0x7F; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java new file mode 100644 index 0000000..bfb9f80 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java @@ -0,0 +1,249 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class GlobalSymbolDictionaryTest { + + @Test + public void testAddSymbol_assignsSequentialIds() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + assertEquals(0, dict.getOrAddSymbol("AAPL")); + assertEquals(1, dict.getOrAddSymbol("GOOG")); + assertEquals(2, dict.getOrAddSymbol("MSFT")); + assertEquals(3, dict.getOrAddSymbol("TSLA")); + + assertEquals(4, dict.size()); + } + + @Test + public void testAddSymbol_deduplicatesSameSymbol() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + int id1 = dict.getOrAddSymbol("AAPL"); + int id2 = dict.getOrAddSymbol("AAPL"); + int id3 = dict.getOrAddSymbol("AAPL"); + + assertEquals(id1, id2); + assertEquals(id2, id3); + assertEquals(0, id1); + assertEquals(1, dict.size()); + } + + @Test + public void testClear() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + dict.getOrAddSymbol("AAPL"); + dict.getOrAddSymbol("GOOG"); + assertEquals(2, dict.size()); + + dict.clear(); + + assertTrue(dict.isEmpty()); + assertEquals(0, dict.size()); + assertFalse(dict.contains("AAPL")); + } + + @Test + public void testClear_thenAddRestartsFromZero() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + dict.getOrAddSymbol("AAPL"); + dict.getOrAddSymbol("GOOG"); + dict.clear(); + + // New IDs should start from 0 + assertEquals(0, dict.getOrAddSymbol("MSFT")); + assertEquals(1, dict.getOrAddSymbol("TSLA")); + } + + @Test + public void testContains() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + assertFalse(dict.contains("AAPL")); + + dict.getOrAddSymbol("AAPL"); + dict.getOrAddSymbol("GOOG"); + + assertTrue(dict.contains("AAPL")); + assertTrue(dict.contains("GOOG")); + assertFalse(dict.contains("MSFT")); + assertFalse(dict.contains(null)); + } + + @Test + public void testCustomInitialCapacity() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(1024); + + // Should work normally + for (int i = 0; i < 100; i++) { + assertEquals(i, dict.getOrAddSymbol("SYM_" + i)); + } + assertEquals(100, dict.size()); + } + + @Test + public void testGetId_returnsCorrectId() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + dict.getOrAddSymbol("AAPL"); + dict.getOrAddSymbol("GOOG"); + dict.getOrAddSymbol("MSFT"); + + assertEquals(0, dict.getId("AAPL")); + assertEquals(1, dict.getId("GOOG")); + assertEquals(2, dict.getId("MSFT")); + } + + @Test + public void testGetId_returnsMinusOneForNull() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + assertEquals(-1, dict.getId(null)); + } + + @Test + public void testGetId_returnsMinusOneForUnknown() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol("AAPL"); + + assertEquals(-1, dict.getId("GOOG")); + assertEquals(-1, dict.getId("UNKNOWN")); + } + + @Test(expected = IllegalArgumentException.class) + public void testGetOrAddSymbol_throwsForNull() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol(null); + } + + @Test + public void testGetSymbol_returnsCorrectSymbol() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + dict.getOrAddSymbol("AAPL"); + dict.getOrAddSymbol("GOOG"); + dict.getOrAddSymbol("MSFT"); + + assertEquals("AAPL", dict.getSymbol(0)); + assertEquals("GOOG", dict.getSymbol(1)); + assertEquals("MSFT", dict.getSymbol(2)); + } + + @Test(expected = IndexOutOfBoundsException.class) + public void testGetSymbol_throwsForInvalidId() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol("AAPL"); + dict.getSymbol(1); // Only id 0 exists + } + + @Test(expected = IndexOutOfBoundsException.class) + public void testGetSymbol_throwsForNegativeId() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol("AAPL"); + dict.getSymbol(-1); + } + + @Test + public void testIsEmpty() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + assertTrue(dict.isEmpty()); + + dict.getOrAddSymbol("AAPL"); + assertFalse(dict.isEmpty()); + } + + @Test + public void testLargeNumberOfSymbols() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + // Add 10000 symbols + for (int i = 0; i < 10000; i++) { + assertEquals(i, dict.getOrAddSymbol("SYMBOL_" + i)); + } + + assertEquals(10000, dict.size()); + + // Verify retrieval + for (int i = 0; i < 10000; i++) { + assertEquals("SYMBOL_" + i, dict.getSymbol(i)); + assertEquals(i, dict.getId("SYMBOL_" + i)); + } + } + + @Test + public void testMixedSymbolsAcrossTables() { + // Simulates symbols from multiple tables sharing the dictionary + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + // Table "trades": exchange column + int nyse = dict.getOrAddSymbol("NYSE"); // 0 + int nasdaq = dict.getOrAddSymbol("NASDAQ"); // 1 + + // Table "prices": currency column + int usd = dict.getOrAddSymbol("USD"); // 2 + int eur = dict.getOrAddSymbol("EUR"); // 3 + + // Table "orders": exchange column (reuses) + int nyse2 = dict.getOrAddSymbol("NYSE"); // Still 0 + + assertEquals(nyse, nyse2); + assertEquals(4, dict.size()); + + // All symbols accessible + assertEquals("NYSE", dict.getSymbol(nyse)); + assertEquals("NASDAQ", dict.getSymbol(nasdaq)); + assertEquals("USD", dict.getSymbol(usd)); + assertEquals("EUR", dict.getSymbol(eur)); + } + + @Test + public void testSpecialCharactersInSymbols() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + dict.getOrAddSymbol(""); // Empty string + dict.getOrAddSymbol(" "); // Space + dict.getOrAddSymbol("a b c"); // With spaces + dict.getOrAddSymbol("AAPL\u0000"); // With null char + dict.getOrAddSymbol("\u00E9"); // Unicode + dict.getOrAddSymbol("\uD83D\uDE00"); // Emoji + + assertEquals(6, dict.size()); + + assertEquals("", dict.getSymbol(0)); + assertEquals(" ", dict.getSymbol(1)); + assertEquals("a b c", dict.getSymbol(2)); + assertEquals("AAPL\u0000", dict.getSymbol(3)); + assertEquals("\u00E9", dict.getSymbol(4)); + assertEquals("\uD83D\uDE00", dict.getSymbol(5)); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InFlightWindowTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InFlightWindowTest.java new file mode 100644 index 0000000..5822d3e --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InFlightWindowTest.java @@ -0,0 +1,843 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.InFlightWindow; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.*; + +/** + * Tests for InFlightWindow. + *

    + * The window assumes sequential batch IDs and cumulative acknowledgments. It + * tracks only the range [lastAcked+1, highestSent] rather than individual batch + * IDs. + */ +public class InFlightWindowTest { + + @Test + public void testAcknowledgeAlreadyAcked() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + window.addInFlight(1); + + // ACK up to 1 + assertTrue(window.acknowledge(1)); + assertTrue(window.isEmpty()); + + // ACK for already acknowledged sequence returns true (idempotent) + assertTrue(window.acknowledge(0)); + assertTrue(window.acknowledge(1)); + assertTrue(window.isEmpty()); + } + + @Test + public void testAcknowledgeUpToAllBatches() { + InFlightWindow window = new InFlightWindow(16, 1000); + + // Add batches + for (int i = 0; i < 10; i++) { + window.addInFlight(i); + } + + // ACK all with high sequence + int acked = window.acknowledgeUpTo(Long.MAX_VALUE); + assertEquals(10, acked); + assertTrue(window.isEmpty()); + } + + @Test + public void testAcknowledgeUpToBasic() { + InFlightWindow window = new InFlightWindow(16, 1000); + + // Add batches 0-9 + for (int i = 0; i < 10; i++) { + window.addInFlight(i); + } + assertEquals(10, window.getInFlightCount()); + + // ACK up to 5 (should remove 0-5, leaving 6-9) + int acked = window.acknowledgeUpTo(5); + assertEquals(6, acked); + assertEquals(4, window.getInFlightCount()); + assertEquals(6, window.getTotalAcked()); + } + + @Test + public void testAcknowledgeUpToEmpty() { + InFlightWindow window = new InFlightWindow(16, 1000); + + // ACK on empty window should be no-op + assertEquals(0, window.acknowledgeUpTo(100)); + assertTrue(window.isEmpty()); + } + + @Test + public void testAcknowledgeUpToIdempotent() { + InFlightWindow window = new InFlightWindow(16, 1000); + + window.addInFlight(0); + window.addInFlight(1); + window.addInFlight(2); + + // First ACK + assertEquals(3, window.acknowledgeUpTo(2)); + assertTrue(window.isEmpty()); + + // Duplicate ACK - should be no-op + assertEquals(0, window.acknowledgeUpTo(2)); + assertTrue(window.isEmpty()); + + // ACK with lower sequence - should be no-op + assertEquals(0, window.acknowledgeUpTo(1)); + assertTrue(window.isEmpty()); + } + + @Test + public void testAcknowledgeUpToWakesAwaitEmpty() throws Exception { + InFlightWindow window = new InFlightWindow(16, 5000); + + window.addInFlight(0); + window.addInFlight(1); + window.addInFlight(2); + + AtomicBoolean waiting = new AtomicBoolean(true); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch finished = new CountDownLatch(1); + + // Start thread waiting for empty + Thread waitThread = new Thread(() -> { + started.countDown(); + window.awaitEmpty(); + waiting.set(false); + finished.countDown(); + }); + waitThread.start(); + + assertTrue(started.await(1, TimeUnit.SECONDS)); + awaitThreadBlocked(waitThread); + assertTrue(waiting.get()); + + // Single cumulative ACK clears all + window.acknowledgeUpTo(2); + + assertTrue(finished.await(1, TimeUnit.SECONDS)); + assertFalse(waiting.get()); + assertTrue(window.isEmpty()); + } + + @Test + public void testAcknowledgeUpToWakesBlockedAdder() throws Exception { + InFlightWindow window = new InFlightWindow(3, 5000); + + // Fill the window + window.addInFlight(0); + window.addInFlight(1); + window.addInFlight(2); + assertTrue(window.isFull()); + + AtomicBoolean blocked = new AtomicBoolean(true); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch finished = new CountDownLatch(1); + + // Start thread that will block + Thread addThread = new Thread(() -> { + started.countDown(); + window.addInFlight(3); + blocked.set(false); + finished.countDown(); + }); + addThread.start(); + + assertTrue(started.await(1, TimeUnit.SECONDS)); + awaitThreadBlocked(addThread); + assertTrue(blocked.get()); + + // Cumulative ACK frees multiple slots + window.acknowledgeUpTo(1); // Removes 0 and 1 + + // Thread should complete + assertTrue(finished.await(1, TimeUnit.SECONDS)); + assertFalse(blocked.get()); + assertEquals(2, window.getInFlightCount()); // batch 2 and 3 + } + + @Test + public void testAwaitEmpty() throws Exception { + InFlightWindow window = new InFlightWindow(8, 5000); + + window.addInFlight(0); + window.addInFlight(1); + window.addInFlight(2); + + AtomicBoolean waiting = new AtomicBoolean(true); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch finished = new CountDownLatch(1); + + // Start thread waiting for empty + Thread waitThread = new Thread(() -> { + started.countDown(); + window.awaitEmpty(); + waiting.set(false); + finished.countDown(); + }); + waitThread.start(); + + assertTrue(started.await(1, TimeUnit.SECONDS)); + awaitThreadBlocked(waitThread); + assertTrue(waiting.get()); + + // Cumulative ACK all batches + window.acknowledgeUpTo(2); + assertTrue(finished.await(1, TimeUnit.SECONDS)); + assertFalse(waiting.get()); + } + + @Test + public void testAwaitEmptyAlreadyEmpty() { + InFlightWindow window = new InFlightWindow(8, 1000); + + // Should return immediately + window.awaitEmpty(); + assertTrue(window.isEmpty()); + } + + @Test + public void testAwaitEmptyTimeout() { + InFlightWindow window = new InFlightWindow(8, 100); // 100ms timeout + + window.addInFlight(0); + + long start = System.currentTimeMillis(); + try { + window.awaitEmpty(); + fail("Expected timeout exception"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("Timeout")); + } + long elapsed = System.currentTimeMillis() - start; + assertTrue("Should have waited at least 100ms", elapsed >= 90); + } + + @Test + public void testBasicAddAndAcknowledge() { + InFlightWindow window = new InFlightWindow(8, 1000); + + assertTrue(window.isEmpty()); + assertEquals(0, window.getInFlightCount()); + + // Add a batch (sequential: 0) + window.addInFlight(0); + assertFalse(window.isEmpty()); + assertEquals(1, window.getInFlightCount()); + + // Acknowledge it (cumulative ACK up to 0) + assertTrue(window.acknowledge(0)); + assertTrue(window.isEmpty()); + assertEquals(0, window.getInFlightCount()); + assertEquals(1, window.getTotalAcked()); + } + + @Test + public void testClearError() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + window.fail(0, new RuntimeException("Test error")); + + assertNotNull(window.getLastError()); + + window.clearError(); + assertNull(window.getLastError()); + + // Should work again + window.addInFlight(1); + assertEquals(2, window.getInFlightCount()); // 0 and 1 both in window (fail doesn't remove) + } + + @Test + public void testConcurrentAddAndAck() throws Exception { + InFlightWindow window = new InFlightWindow(4, 5000); + int numOperations = 100; + CountDownLatch done = new CountDownLatch(2); + AtomicReference error = new AtomicReference<>(); + AtomicInteger highestAdded = new AtomicInteger(-1); + + // Sender thread + Thread sender = new Thread(() -> { + try { + for (int i = 0; i < numOperations; i++) { + window.addInFlight(i); + highestAdded.set(i); + Thread.sleep(1); // Small delay + } + } catch (Throwable t) { + error.set(t); + } finally { + done.countDown(); + } + }); + + // ACK thread (cumulative ACKs) + Thread acker = new Thread(() -> { + try { + Thread.sleep(10); // Let sender get ahead + int lastAcked = -1; + while (lastAcked < numOperations - 1) { + int highest = highestAdded.get(); + if (highest > lastAcked) { + window.acknowledgeUpTo(highest); + lastAcked = highest; + } + Thread.sleep(1); + } + } catch (Throwable t) { + error.set(t); + } finally { + done.countDown(); + } + }); + + sender.start(); + acker.start(); + + assertTrue(done.await(10, TimeUnit.SECONDS)); + assertNull(error.get()); + assertTrue(window.isEmpty()); + assertEquals(numOperations, window.getTotalAcked()); + } + + @Test + public void testConcurrentAddAndCumulativeAck() throws Exception { + InFlightWindow window = new InFlightWindow(100, 10000); + int numBatches = 500; + CountDownLatch done = new CountDownLatch(2); + AtomicReference error = new AtomicReference<>(); + AtomicInteger highestAdded = new AtomicInteger(-1); + + // Sender thread + Thread sender = new Thread(() -> { + try { + for (int i = 0; i < numBatches; i++) { + window.addInFlight(i); + highestAdded.set(i); + } + } catch (Throwable t) { + error.set(t); + } finally { + done.countDown(); + } + }); + + // ACK thread using cumulative ACKs + Thread acker = new Thread(() -> { + try { + int lastAcked = -1; + while (lastAcked < numBatches - 1) { + int highest = highestAdded.get(); + if (highest > lastAcked) { + window.acknowledgeUpTo(highest); + lastAcked = highest; + } else { + Thread.sleep(1); + } + } + } catch (Throwable t) { + error.set(t); + } finally { + done.countDown(); + } + }); + + sender.start(); + acker.start(); + + assertTrue(done.await(30, TimeUnit.SECONDS)); + assertNull(error.get()); + assertTrue(window.isEmpty()); + assertEquals(numBatches, window.getTotalAcked()); + } + + @Test + public void testDefaultWindowSize() { + InFlightWindow window = new InFlightWindow(); + assertEquals(InFlightWindow.DEFAULT_WINDOW_SIZE, window.getMaxWindowSize()); + } + + @Test + public void testFailAllPropagatesError() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + window.addInFlight(1); + window.failAll(new RuntimeException("Transport down")); + + try { + window.awaitEmpty(); + fail("Expected exception due to failAll"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("failed")); + assertTrue(e.getMessage().contains("Transport down")); + } + } + + @Test + public void testFailBatch() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + window.addInFlight(1); + + // Fail batch 0 + RuntimeException error = new RuntimeException("Test error"); + window.fail(0, error); + + assertEquals(1, window.getTotalFailed()); + assertNotNull(window.getLastError()); + } + + @Test + public void testFailPropagatesError() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + window.fail(0, new RuntimeException("Test error")); + + // Subsequent operations should throw + try { + window.addInFlight(1); + fail("Expected exception due to error"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("failed")); + } + + try { + window.awaitEmpty(); + fail("Expected exception due to error"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("failed")); + } + } + + @Test + public void testFailThenClearThenAdd() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + window.fail(0, new RuntimeException("Error")); + + // Should not be able to add + try { + window.addInFlight(1); + fail("Expected exception"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("failed")); + } + + // Clear error + window.clearError(); + + // Should work now + window.addInFlight(1); + assertEquals(2, window.getInFlightCount()); + } + + @Test + public void testFailWakesAwaitEmpty() throws Exception { + InFlightWindow window = new InFlightWindow(8, 5000); + + window.addInFlight(0); + + CountDownLatch started = new CountDownLatch(1); + AtomicReference caught = new AtomicReference<>(); + + // Thread waiting for empty + Thread waitThread = new Thread(() -> { + started.countDown(); + try { + window.awaitEmpty(); + } catch (LineSenderException e) { + caught.set(e); + } + }); + waitThread.start(); + + assertTrue(started.await(1, TimeUnit.SECONDS)); + awaitThreadBlocked(waitThread); + + // Fail a batch - should wake the blocked thread + window.fail(0, new RuntimeException("Test error")); + + waitThread.join(1000); + assertFalse(waitThread.isAlive()); + assertNotNull(caught.get()); + assertTrue(caught.get().getMessage().contains("failed")); + } + + @Test + public void testFailWakesBlockedAdder() throws Exception { + InFlightWindow window = new InFlightWindow(2, 5000); + + // Fill the window + window.addInFlight(0); + window.addInFlight(1); + + CountDownLatch started = new CountDownLatch(1); + AtomicReference caught = new AtomicReference<>(); + + // Thread that will block on add + Thread addThread = new Thread(() -> { + started.countDown(); + try { + window.addInFlight(2); + } catch (LineSenderException e) { + caught.set(e); + } + }); + addThread.start(); + + assertTrue(started.await(1, TimeUnit.SECONDS)); + awaitThreadBlocked(addThread); + + // Fail a batch - should wake the blocked thread + window.fail(0, new RuntimeException("Test error")); + + addThread.join(1000); + assertFalse(addThread.isAlive()); + assertNotNull(caught.get()); + assertTrue(caught.get().getMessage().contains("failed")); + } + + @Test + public void testFillAndDrainRepeatedly() { + InFlightWindow window = new InFlightWindow(4, 1000); + + int batchId = 0; + for (int cycle = 0; cycle < 100; cycle++) { + // Fill + int startBatch = batchId; + for (int i = 0; i < 4; i++) { + window.addInFlight(batchId++); + } + assertTrue(window.isFull()); + assertEquals(4, window.getInFlightCount()); + + // Drain with cumulative ACK + window.acknowledgeUpTo(batchId - 1); + assertTrue(window.isEmpty()); + } + + assertEquals(400, window.getTotalAcked()); + } + + @Test + public void testGetMaxWindowSize() { + InFlightWindow window = new InFlightWindow(16, 1000); + assertEquals(16, window.getMaxWindowSize()); + } + + @Test + public void testHasWindowSpace() { + InFlightWindow window = new InFlightWindow(2, 1000); + + assertTrue(window.hasWindowSpace()); + window.addInFlight(0); + assertTrue(window.hasWindowSpace()); + window.addInFlight(1); + assertFalse(window.hasWindowSpace()); + + window.acknowledge(0); + assertTrue(window.hasWindowSpace()); + } + + @Test + public void testHighConcurrencyStress() throws Exception { + InFlightWindow window = new InFlightWindow(8, 30000); + int numBatches = 10000; + CountDownLatch done = new CountDownLatch(2); + AtomicReference error = new AtomicReference<>(); + AtomicInteger highestAdded = new AtomicInteger(-1); + + // Fast sender thread + Thread sender = new Thread(() -> { + try { + for (int i = 0; i < numBatches; i++) { + window.addInFlight(i); + highestAdded.set(i); + } + } catch (Throwable t) { + error.set(t); + } finally { + done.countDown(); + } + }); + + // Fast ACK thread + Thread acker = new Thread(() -> { + try { + int lastAcked = -1; + while (lastAcked < numBatches - 1) { + int highest = highestAdded.get(); + if (highest > lastAcked) { + window.acknowledgeUpTo(highest); + lastAcked = highest; + } else { + Thread.sleep(1); + } + } + } catch (Throwable t) { + error.set(t); + } finally { + done.countDown(); + } + }); + + sender.start(); + acker.start(); + + assertTrue(done.await(60, TimeUnit.SECONDS)); + if (error.get() != null) { + error.get().printStackTrace(); + } + assertNull(error.get()); + assertTrue(window.isEmpty()); + assertEquals(numBatches, window.getTotalAcked()); + } + + @Test(expected = IllegalArgumentException.class) + public void testInvalidWindowSize() { + new InFlightWindow(0, 1000); + } + + @Test + public void testMultipleBatches() { + InFlightWindow window = new InFlightWindow(8, 1000); + + // Add sequential batches 0-4 + for (long i = 0; i < 5; i++) { + window.addInFlight(i); + } + assertEquals(5, window.getInFlightCount()); + + // Cumulative ACK up to 2 (acknowledges 0, 1, 2) + assertEquals(3, window.acknowledgeUpTo(2)); + assertEquals(2, window.getInFlightCount()); + + // Cumulative ACK up to 4 (acknowledges 3, 4) + assertEquals(2, window.acknowledgeUpTo(4)); + assertTrue(window.isEmpty()); + assertEquals(5, window.getTotalAcked()); + } + + @Test + public void testMultipleResets() { + InFlightWindow window = new InFlightWindow(8, 1000); + + for (int cycle = 0; cycle < 10; cycle++) { + window.addInFlight(cycle); + window.reset(); + + assertTrue(window.isEmpty()); + assertNull(window.getLastError()); + } + } + + @Test + public void testRapidAddAndAck() { + InFlightWindow window = new InFlightWindow(16, 5000); + + // Rapid add and ack in same thread + for (int i = 0; i < 10000; i++) { + window.addInFlight(i); + assertTrue(window.acknowledge(i)); + } + + assertTrue(window.isEmpty()); + assertEquals(10000, window.getTotalAcked()); + } + + @Test + public void testReset() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + window.addInFlight(1); + window.fail(2, new RuntimeException("Test")); + + window.reset(); + + assertTrue(window.isEmpty()); + assertNull(window.getLastError()); + assertEquals(0, window.getInFlightCount()); + } + + @Test + public void testSmallestPossibleWindow() { + InFlightWindow window = new InFlightWindow(1, 1000); + + window.addInFlight(0); + assertTrue(window.isFull()); + + window.acknowledge(0); + assertFalse(window.isFull()); + } + + @Test + public void testTryAddInFlight() { + InFlightWindow window = new InFlightWindow(2, 1000); + + // Should succeed + assertTrue(window.tryAddInFlight(0)); + assertTrue(window.tryAddInFlight(1)); + + // Should fail - window full + assertFalse(window.tryAddInFlight(2)); + + // After ACK, should succeed + window.acknowledge(0); + assertTrue(window.tryAddInFlight(2)); + } + + @Test + public void testVeryLargeWindow() { + InFlightWindow window = new InFlightWindow(10000, 1000); + + // Add many batches + for (int i = 0; i < 5000; i++) { + window.addInFlight(i); + } + assertEquals(5000, window.getInFlightCount()); + assertFalse(window.isFull()); + + // ACK half + window.acknowledgeUpTo(2499); + assertEquals(2500, window.getInFlightCount()); + } + + @Test + public void testWindowBlocksTimeout() { + InFlightWindow window = new InFlightWindow(2, 100); // 100ms timeout + + // Fill the window + window.addInFlight(0); + window.addInFlight(1); + + // Try to add another - should timeout + long start = System.currentTimeMillis(); + try { + window.addInFlight(2); + fail("Expected timeout exception"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("Timeout")); + } + long elapsed = System.currentTimeMillis() - start; + assertTrue("Should have waited at least 100ms", elapsed >= 90); + } + + @Test + public void testWindowBlocksWhenFull() throws Exception { + InFlightWindow window = new InFlightWindow(2, 5000); + + // Fill the window + window.addInFlight(0); + window.addInFlight(1); + + AtomicBoolean blocked = new AtomicBoolean(true); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch finished = new CountDownLatch(1); + + // Start thread that will block + Thread addThread = new Thread(() -> { + started.countDown(); + window.addInFlight(2); + blocked.set(false); + finished.countDown(); + }); + addThread.start(); + + // Wait for thread to start and block + assertTrue(started.await(1, TimeUnit.SECONDS)); + awaitThreadBlocked(addThread); + assertTrue(blocked.get()); + + // Free a slot + window.acknowledge(0); + + // Thread should complete + assertTrue(finished.await(1, TimeUnit.SECONDS)); + assertFalse(blocked.get()); + assertEquals(2, window.getInFlightCount()); + } + + @Test + public void testWindowFull() { + InFlightWindow window = new InFlightWindow(3, 1000); + + // Fill the window + window.addInFlight(0); + window.addInFlight(1); + window.addInFlight(2); + + assertTrue(window.isFull()); + assertEquals(3, window.getInFlightCount()); + + // Free slots by ACKing + window.acknowledgeUpTo(1); + assertFalse(window.isFull()); + assertEquals(1, window.getInFlightCount()); + } + + @Test + public void testZeroBatchId() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + assertEquals(1, window.getInFlightCount()); + + assertTrue(window.acknowledge(0)); + assertTrue(window.isEmpty()); + } + + private static void awaitThreadBlocked(Thread thread) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (System.nanoTime() < deadline) { + Thread.State state = thread.getState(); + if (state == Thread.State.WAITING || state == Thread.State.TIMED_WAITING) { + return; + } + Thread.sleep(1); + } + fail("Thread did not reach blocked state within 5s, state: " + thread.getState()); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderUdpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderUdpTest.java new file mode 100644 index 0000000..5e8f191 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderUdpTest.java @@ -0,0 +1,373 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.QwpUdpSender; +import io.questdb.client.test.AbstractTest; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +/** + * Tests for UDP transport support in the Sender.builder() API. + * These tests verify the builder configuration and validation, + * not actual UDP connectivity (which requires a running server). + */ +public class LineSenderBuilderUdpTest extends AbstractTest { + + @Test + public void testInvalidSchema_includesUdp() { + assertBadConfig("invalid::addr=localhost:9000;", "udp"); + } + + @Test + public void testUdpScheme_buildsQwpUdpSender() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (Sender sender = Sender.fromConfig("udp::addr=localhost:9007;")) { + Assert.assertTrue(sender instanceof QwpUdpSender); + } + }); + } + + @Test + public void testUdpScheme_customMaxDatagramSize() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (Sender sender = Sender.fromConfig("udp::addr=localhost:9007;max_datagram_size=500;")) { + Assert.assertTrue(sender instanceof QwpUdpSender); + } + }); + } + + @Test + public void testUdpScheme_customMulticastTtl() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (Sender sender = Sender.fromConfig("udp::addr=localhost:9007;multicast_ttl=5;")) { + Assert.assertTrue(sender instanceof QwpUdpSender); + } + }); + } + + @Test + public void testUdpScheme_noAddr_throws() { + assertBadConfig("udp::foo=bar;", "addr is missing"); + } + + @Test + public void testUdp_authNotSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.UDP) + .address("localhost") + .enableAuth("keyId") + .authToken("5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48"), + "not supported for UDP"); + } + + @Test + public void testUdp_autoFlushBytesNotSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.UDP) + .address("localhost") + .autoFlushBytes(1000), + "not supported for UDP"); + } + + @Test + public void testUdp_autoFlushIntervalNotSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.UDP) + .address("localhost") + .autoFlushIntervalMillis(100), + "not supported for UDP"); + } + + @Test + public void testUdp_autoFlushRowsNotSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.UDP) + .address("localhost") + .autoFlushRows(100), + "not supported for UDP"); + } + + @Test + public void testUdp_defaultPort9007() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (Sender sender = Sender.fromConfig("udp::addr=localhost;")) { + Assert.assertTrue(sender instanceof QwpUdpSender); + } + }); + } + + @Test + public void testUdp_httpPathNotSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.UDP) + .address("localhost") + .httpPath("/custom/path"), + "not supported for UDP"); + } + + @Test + public void testUdp_httpTimeoutNotSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.UDP) + .address("localhost") + .httpTimeoutMillis(5000), + "not supported for UDP"); + } + + @Test + public void testUdp_httpTokenNotSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.UDP) + .address("localhost") + .httpToken("token"), + "not supported for UDP"); + } + + @Test + public void testUdp_inFlightWindowSizeNotSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.UDP) + .address("localhost") + .inFlightWindowSize(1000), + "not supported for UDP"); + } + + @Test + public void testUdp_maxBackoffNotSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.UDP) + .address("localhost") + .maxBackoffMillis(5000), + "not supported for UDP"); + } + + @Test + public void testUdp_maxDatagramSize() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.UDP) + .address("localhost") + .maxDatagramSize(500); + Assert.assertNotNull(builder); + } + + @Test + public void testUdp_maxDatagramSizeDoubleSet_fails() { + assertThrows("already configured", + () -> Sender.builder(Sender.Transport.UDP) + .address("localhost") + .maxDatagramSize(500) + .maxDatagramSize(600)); + } + + @Test + public void testUdp_maxDatagramSizeZero_fails() { + assertThrows("must be positive", + () -> Sender.builder(Sender.Transport.UDP) + .address("localhost") + .maxDatagramSize(0)); + } + + @Test + public void testUdp_minRequestThroughputNotSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.UDP) + .address("localhost") + .minRequestThroughput(100), + "not supported for UDP"); + } + + @Test + public void testUdp_multicastTtl() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.UDP) + .address("localhost") + .multicastTtl(5); + Assert.assertNotNull(builder); + } + + @Test + public void testUdp_multicastTtlDoubleSet_fails() { + assertThrows("already configured", + () -> Sender.builder(Sender.Transport.UDP) + .address("localhost") + .multicastTtl(5) + .multicastTtl(10)); + } + + @Test + public void testUdp_multicastTtlNegative_fails() { + assertThrows("cannot be negative", + () -> Sender.builder(Sender.Transport.UDP) + .address("localhost") + .multicastTtl(-1)); + } + + @Test + public void testUdp_protocolVersionNotSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.UDP) + .address("localhost") + .protocolVersion(1), + "not supported for UDP"); + } + + @Test + public void testUdp_ipv6Address_throws() { + assertThrowsAny( + () -> Sender.builder(Sender.Transport.UDP) + .address("::1"), + "cannot parse a port", "use IPv4 address"); + } + + @Test + public void testUdp_resolveUnknownHost_throws() { + assertThrowsAny( + Sender.builder(Sender.Transport.UDP) + .address("this.host.does.not.exist.questdb.invalid"), + "could not resolve host"); + } + + @Test + public void testUdp_retryTimeoutNotSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.UDP) + .address("localhost") + .retryTimeoutMillis(100), + "not supported for UDP"); + } + + @Test + public void testUdp_tlsEnabled_throws() { + assertThrowsAny( + Sender.builder(Sender.Transport.UDP) + .address("localhost") + .enableTls(), + "TLS is not supported for UDP"); + } + + @Test + public void testUdpScheme_inFlightWindow_fails() { + assertBadConfig("udp::addr=localhost:9007;in_flight_window=64;", "only supported for WebSocket"); + } + + @Test + public void testUdp_tokenNotSupported() { + assertBadConfig("udp::addr=localhost:9007;token=foo;", "token is not supported for UDP"); + } + + @Test + public void testUdp_transportEnum() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (Sender sender = Sender.builder(Sender.Transport.UDP) + .address("localhost:9007") + .build()) { + Assert.assertTrue(sender instanceof QwpUdpSender); + } + }); + } + + @Test + public void testUdp_usernamePasswordNotSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.UDP) + .address("localhost") + .httpUsernamePassword("user", "pass"), + "not supported for UDP"); + } + + @Test + public void testUdpScheme_password_throws() { + assertBadConfig("udp::addr=localhost;password=bar;", "password is not supported for UDP"); + } + + @Test + public void testUdpScheme_username_throws() { + assertBadConfig("udp::addr=localhost;username=foo;", "username is not supported for UDP"); + } + + @Test + public void testUdp_maxDatagramSizeNonUdp_fails() { + assertThrows("only supported for UDP transport", + () -> Sender.builder(Sender.Transport.HTTP) + .address("localhost") + .maxDatagramSize(500)); + } + + @Test + public void testUdp_multicastTtlExceeds255_fails() { + assertThrows("cannot exceed 255", + () -> Sender.builder(Sender.Transport.UDP) + .address("localhost") + .multicastTtl(256)); + } + + @Test + public void testUdp_multicastTtlNonUdp_fails() { + assertThrows("only supported for UDP transport", + () -> Sender.builder(Sender.Transport.HTTP) + .address("localhost") + .multicastTtl(1)); + } + + @Test + public void testUdps_throws() { + assertBadConfig("udps::addr=localhost:9007;", "TLS is not supported for UDP"); + } + + @SuppressWarnings("resource") + private static void assertBadConfig(String config, String... anyOf) { + assertThrowsAny(() -> Sender.fromConfig(config), anyOf); + } + + private static void assertThrows(String expectedSubstring, Runnable action) { + try { + action.run(); + Assert.fail("Expected LineSenderException containing '" + expectedSubstring + "'"); + } catch (LineSenderException e) { + TestUtils.assertContains(e.getMessage(), expectedSubstring); + } + } + + private static void assertThrowsAny(Sender.LineSenderBuilder builder, String... anyOf) { + assertThrowsAny(builder::build, anyOf); + } + + private static void assertThrowsAny(Runnable action, String... anyOf) { + try { + action.run(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + for (String s : anyOf) { + if (msg.contains(s)) { + return; + } + } + Assert.fail("Expected message containing one of [" + String.join(", ", anyOf) + "] but got: " + msg); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java new file mode 100644 index 0000000..1505f1c --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java @@ -0,0 +1,710 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.test.AbstractTest; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +/** + * Tests for WebSocket transport support in the Sender.builder() API. + * These tests verify the builder configuration and validation, + * not actual WebSocket connectivity (which requires a running server). + */ +public class LineSenderBuilderWebSocketTest extends AbstractTest { + + private static final String LOCALHOST = "localhost"; + + @Test + public void testAddressConfiguration() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST + ":9000"); + Assert.assertNotNull(builder); + } + + @Test + public void testAddressEmpty_fails() { + assertThrows("address cannot be empty", + () -> Sender.builder(Sender.Transport.WEBSOCKET).address("")); + } + + @Test + public void testAddressEndsWithColon_fails() { + assertThrows("invalid address", + () -> Sender.builder(Sender.Transport.WEBSOCKET).address("foo:")); + } + + @Test + public void testAddressNull_fails() { + assertThrows("null", + () -> Sender.builder(Sender.Transport.WEBSOCKET).address(null)); + } + + @Test + public void testAddressWithoutPort_usesDefaultPort9000() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST); + Assert.assertNotNull(builder); + } + + @Test + public void testAsyncModeWithAllOptions() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushRows(500) + .autoFlushBytes(512 * 1024) + .autoFlushIntervalMillis(50) + .inFlightWindowSize(8); + Assert.assertNotNull(builder); + } + + @Test + public void testAutoFlushBytes() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushBytes(1024 * 1024); + Assert.assertNotNull(builder); + } + + @Test + public void testAutoFlushBytesDoubleSet_fails() { + assertThrows("already configured", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushBytes(1024) + .autoFlushBytes(2048)); + } + + @Test + public void testAutoFlushBytesNegative_fails() { + assertThrows("cannot be negative", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushBytes(-1)); + } + + @Test + public void testAutoFlushBytesZero() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushBytes(0); + Assert.assertNotNull(builder); + } + + @Test + public void testAutoFlushIntervalMillis() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushIntervalMillis(100); + Assert.assertNotNull(builder); + } + + @Test + public void testAutoFlushIntervalMillisDoubleSet_fails() { + assertThrows("already configured", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushIntervalMillis(100) + .autoFlushIntervalMillis(200)); + } + + @Test + public void testAutoFlushIntervalMillisNegative_fails() { + assertThrows("cannot be negative", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushIntervalMillis(-1)); + } + + @Test + public void testAutoFlushIntervalMillisZero_fails() { + assertThrows("cannot be negative", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushIntervalMillis(0)); + } + + @Test + public void testAutoFlushRows() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushRows(1000); + Assert.assertNotNull(builder); + } + + @Test + public void testAutoFlushRowsDoubleSet_fails() { + assertThrows("already configured", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushRows(100) + .autoFlushRows(200)); + } + + @Test + public void testAutoFlushRowsNegative_fails() { + assertThrows("cannot be negative", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushRows(-1)); + } + + @Test + public void testAutoFlushRowsZero_disablesRowBasedAutoFlush() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushRows(0); + Assert.assertNotNull(builder); + } + + @Test + public void testBufferCapacity() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .bufferCapacity(128 * 1024); + Assert.assertNotNull(builder); + } + + @Test + public void testBufferCapacityDoubleSet_fails() { + assertThrows("already configured", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .bufferCapacity(1024) + .bufferCapacity(2048)); + } + + @Test + public void testBufferCapacityNegative_fails() { + assertThrows("cannot be negative", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .bufferCapacity(-1)); + } + + @Test + public void testBuilderWithWebSocketTransport() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET); + Assert.assertNotNull("Builder should be created for WebSocket transport", builder); + } + + @Test + public void testBuilderWithWebSocketTransportCreatesCorrectSenderType() throws Exception { + assertMemoryLeak(() -> { + int port; + try (java.net.ServerSocket s = new java.net.ServerSocket(0)) { + port = s.getLocalPort(); + } + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST + ":" + port), + "connect", "Failed" + ); + }); + } + + @Test + public void testConnectionRefused() throws Exception { + assertMemoryLeak(() -> { + int port = findUnusedPort(); + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST + ":" + port), + "connect", "Failed" + ); + }); + } + + @Test + public void testCustomTrustStore_butTlsNotEnabled_fails() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .advancedTls().customTrustStore("/some/path", "password".toCharArray()) + .address(LOCALHOST), + "TLS was not enabled"); + } + + @Test + public void testDisableAutoFlush_notSupportedForWebSocket() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .disableAutoFlush(), + "not supported for WebSocket"); + } + + @Test + public void testDnsResolutionFailure() throws Exception { + assertMemoryLeak(() -> { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address("this-domain-does-not-exist-i-hope-better-to-use-a-silly-tld.silly-tld:9000"), + "resolve", "connect", "Failed" + ); + }); + } + + @Test + public void testDuplicateAddresses_fails() { + assertThrows("duplicated addresses", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST + ":9000") + .address(LOCALHOST + ":9000")); + } + + @Test + @Ignore("TCP authentication is not supported for WebSocket protocol") + public void testEnableAuth_notSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .enableAuth("keyId") + .authToken("token"), + "not supported for WebSocket"); + } + + @Test + public void testFullAsyncConfiguration() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushRows(1000) + .autoFlushBytes(1024 * 1024) + .autoFlushIntervalMillis(100) + .inFlightWindowSize(16); + Assert.assertNotNull(builder); + } + + @Test + public void testFullAsyncConfigurationWithTls() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .enableTls() + .advancedTls().disableCertificateValidation() + .autoFlushRows(1000) + .autoFlushBytes(1024 * 1024) + .inFlightWindowSize(16); + Assert.assertNotNull(builder); + } + + @Test + public void testHttpPath_notSupportedForWebSocket() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .httpPath("/custom/path"), + "not supported for WebSocket"); + } + + @Test + public void testHttpTimeout_notSupportedForWebSocket() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .httpTimeoutMillis(5000), + "not supported for WebSocket"); + } + + @Test + public void testHttpToken_accepted() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .httpToken("token"); + Assert.assertNotNull(builder); + } + + @Test + public void testInFlightWindowSizeDoubleSet_fails() { + assertThrows("already configured", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .inFlightWindowSize(8) + .inFlightWindowSize(16)); + } + + @Test + public void testInFlightWindowSizeNegative_fails() { + assertThrows("must be positive", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .inFlightWindowSize(-1)); + } + + @Test + public void testInFlightWindowSizeOne_syncMode() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .inFlightWindowSize(1); + Assert.assertNotNull(builder); + } + + @Test + public void testInFlightWindowSizeZero_fails() { + assertThrows("must be positive", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .inFlightWindowSize(0)); + } + + @Test + public void testInFlightWindowSize_customValue() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .inFlightWindowSize(16); + Assert.assertNotNull(builder); + } + + @Test + public void testInvalidPort_fails() { + assertThrows("invalid port", + () -> Sender.builder(Sender.Transport.WEBSOCKET).address(LOCALHOST + ":99999")); + } + + @Test + public void testInvalidSchema_fails() { + assertBadConfig("invalid::addr=localhost:9000;", "invalid schema [schema=invalid, supported-schemas=[http, https, tcp, tcps, ws, wss, udp]]"); + } + + @Test + public void testMalformedPortInAddress_fails() { + assertThrows("cannot parse a port from the address", + () -> Sender.builder(Sender.Transport.WEBSOCKET).address("foo:nonsense12334")); + } + + @Test + public void testMaxBackoff_notSupportedForWebSocket() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .maxBackoffMillis(1000), + "not supported for WebSocket"); + } + + @Test + public void testMaxNameLength() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .maxNameLength(256); + Assert.assertNotNull(builder); + } + + @Test + public void testMaxNameLengthDoubleSet_fails() { + assertThrows("already configured", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .maxNameLength(128) + .maxNameLength(256)); + } + + @Test + public void testMaxNameLengthTooSmall_fails() { + assertThrows("at least 16 bytes", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .maxNameLength(10)); + } + + @Test + public void testMinRequestThroughput_notSupportedForWebSocket() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .minRequestThroughput(10000), + "not supported for WebSocket"); + } + + @Test + public void testMultipleAddresses_fails() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST + ":9000") + .address(LOCALHOST + ":9001"), + "single address"); + } + + @Test + public void testNoAddress_fails() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET), + "address not set"); + } + + @Test + public void testPortMismatch_fails() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST + ":9000") + .port(9001), + "mismatch"); + } + + @Test + public void testProtocolVersion_notSupportedForWebSocket() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .protocolVersion(Sender.PROTOCOL_VERSION_V2), + "not supported for WebSocket"); + } + + @Test + public void testRetryTimeout_notSupportedForWebSocket() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .retryTimeoutMillis(5000), + "not supported for WebSocket"); + } + + @Test + public void testSyncModeAutoFlushDefaults() throws Exception { + // Regression test: sync-mode connect() must not hardcode autoFlush to 0. + // createForTesting(host, port, windowSize) mirrors what connect(h,p,tls) + // creates internally. Verify it uses sensible defaults. + assertMemoryLeak(() -> { + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 0, 1); + try { + Assert.assertEquals( + QwpWebSocketSender.DEFAULT_AUTO_FLUSH_ROWS, + sender.getAutoFlushRows() + ); + Assert.assertEquals( + QwpWebSocketSender.DEFAULT_AUTO_FLUSH_BYTES, + sender.getAutoFlushBytes() + ); + Assert.assertEquals( + QwpWebSocketSender.DEFAULT_AUTO_FLUSH_INTERVAL_NANOS, + sender.getAutoFlushIntervalNanos() + ); + } finally { + sender.close(); + } + }); + } + + @Test + public void testDefaultIsAsync() { + // Default in-flight window size is 128 (async) + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST); + Assert.assertNotNull(builder); + } + + @Test + public void testTcpAuth_fails() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .enableAuth("keyId") + .authToken("5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48"), + "not supported for WebSocket"); + } + + @Test + public void testTlsDoubleSet_fails() { + assertThrows("already enabled", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .enableTls() + .enableTls()); + } + + @Test + public void testTlsEnabled() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .enableTls(); + Assert.assertNotNull(builder); + } + + @Test + public void testTlsValidationDisabled() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .enableTls() + .advancedTls().disableCertificateValidation(); + Assert.assertNotNull(builder); + } + + @Test + public void testTlsValidationDisabled_butTlsNotEnabled_fails() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .advancedTls().disableCertificateValidation() + .address(LOCALHOST), + "TLS was not enabled"); + } + + @Test + public void testUsernamePassword_accepted() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .httpUsernamePassword("user", "pass"); + Assert.assertNotNull(builder); + } + + @Test + public void testWsConfigString_inFlightWindow() throws Exception { + assertMemoryLeak(() -> { + int port = findUnusedPort(); + assertBadConfig("ws::addr=localhost:" + port + ";in_flight_window=64;", "connect", "Failed"); + }); + } + + @Test + public void testWsConfigString_inFlightWindowDoubleSet_fails() { + assertBadConfig("ws::addr=localhost:9000;in_flight_window=64;in_flight_window=128;", "already configured"); + } + + @Test + public void testWsConfigString_inFlightWindowInvalid_fails() { + assertBadConfig("ws::addr=localhost:9000;in_flight_window=0;", "must be positive"); + } + + @Test + public void testWsConfigString_inFlightWindowNotSupportedForHttp_fails() { + assertBadConfig("http::addr=localhost:9000;in_flight_window=64;", "only supported for WebSocket"); + } + + @Test + public void testWsConfigString_inFlightWindowSync() throws Exception { + assertMemoryLeak(() -> { + int port = findUnusedPort(); + assertBadConfig("ws::addr=localhost:" + port + ";in_flight_window=1;", "connect", "Failed"); + }); + } + + @Test + public void testWsConfigString() throws Exception { + assertMemoryLeak(() -> { + int port = findUnusedPort(); + assertBadConfig("ws::addr=localhost:" + port + ";", "connect", "Failed"); + }); + } + + @Test + public void testWsConfigString_missingAddr_fails() throws Exception { + assertMemoryLeak(() -> { + int port = findUnusedPort(); + assertBadConfig("ws::addr=localhost:" + port + ";", "connect", "Failed"); + assertBadConfig("ws::foo=bar;", "addr is missing"); + }); + } + + @Test + public void testWsConfigString_protocolAlreadyConfigured_fails() throws Exception { + assertMemoryLeak(() -> { + int port = findUnusedPort(); + assertThrowsAny( + Sender.builder("ws::addr=localhost:" + port + ";") + .enableTls(), + "TLS", "connect", "Failed" + ); + }); + } + + @Test + public void testWsConfigString_uppercaseNotSupported() { + assertBadConfig("WS::addr=localhost:9000;", "invalid schema"); + } + + @Test + public void testWsConfigString_withToken() throws Exception { + assertMemoryLeak(() -> { + int port = findUnusedPort(); + assertBadConfig("ws::addr=localhost:" + port + ";token=mytoken;", "token is not supported"); + }); + } + + @Test + public void testWsConfigString_withUsernamePassword() throws Exception { + assertMemoryLeak(() -> { + int port = findUnusedPort(); + assertBadConfig("ws::addr=localhost:" + port + ";username=user;password=pass;", "connect", "Failed"); + }); + } + + @Test + public void testWssConfigString() throws Exception { + assertMemoryLeak(() -> { + assertBadConfig("wss::addr=localhost:9000;tls_verify=unsafe_off;", "connect", "Failed", "SSL"); + }); + } + + @Test + public void testWssConfigString_uppercaseNotSupported() { + assertBadConfig("WSS::addr=localhost:9000;", "invalid schema"); + } + + @SuppressWarnings("resource") + private static void assertBadConfig(String config, String... anyOf) { + assertThrowsAny(() -> Sender.fromConfig(config), anyOf); + } + + private static void assertThrows(String expectedSubstring, Runnable action) { + try { + action.run(); + Assert.fail("Expected LineSenderException containing '" + expectedSubstring + "'"); + } catch (LineSenderException e) { + TestUtils.assertContains(e.getMessage(), expectedSubstring); + } + } + + private static void assertThrowsAny(Sender.LineSenderBuilder builder, String... anyOf) { + assertThrowsAny(builder::build, anyOf); + } + + private static void assertThrowsAny(Runnable action, String... anyOf) { + try { + action.run(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + for (String s : anyOf) { + if (msg.contains(s)) { + return; + } + } + Assert.fail("Expected message containing one of [" + String.join(", ", anyOf) + "] but got: " + msg); + } + } + + // There is a TOCTOU race between closing the ServerSocket and the caller's + // connect attempt — another process could bind the port in between. This is + // acceptable because every caller is a negative test that expects the connection + // to fail. If the port is stolen, the test connects to a non-QuestDB endpoint, + // which also fails with the same error. + private static int findUnusedPort() throws Exception { + try (java.net.ServerSocket s = new java.net.ServerSocket(0)) { + return s.getLocalPort(); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java new file mode 100644 index 0000000..bda1b1f --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java @@ -0,0 +1,695 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.MicrobatchBuffer; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class MicrobatchBufferTest { + + @Test + public void testAwaitRecycled() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.seal(); + buffer.markSending(); + + AtomicBoolean recycled = new AtomicBoolean(false); + CountDownLatch started = new CountDownLatch(1); + + Thread waiter = new Thread(() -> { + started.countDown(); + buffer.awaitRecycled(); + recycled.set(true); + }); + waiter.start(); + + started.await(); + Thread.sleep(50); // Give waiter time to start waiting + Assert.assertFalse(recycled.get()); + + buffer.markRecycled(); + waiter.join(1000); + + Assert.assertTrue(recycled.get()); + } + }); + } + + @Test + public void testAwaitRecycledWithTimeout() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.seal(); + buffer.markSending(); + + // Should timeout + boolean result = buffer.awaitRecycled(50, TimeUnit.MILLISECONDS); + Assert.assertFalse(result); + + buffer.markRecycled(); + + // Should succeed immediately now + result = buffer.awaitRecycled(50, TimeUnit.MILLISECONDS); + Assert.assertTrue(result); + } + }); + } + + @Test + public void testBatchIdIncrementsOnReset() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + long id1 = buffer.getBatchId(); + + buffer.seal(); + buffer.markSending(); + buffer.markRecycled(); + buffer.reset(); + + long id2 = buffer.getBatchId(); + Assert.assertNotEquals(id1, id2); + + buffer.seal(); + buffer.markSending(); + buffer.markRecycled(); + buffer.reset(); + + long id3 = buffer.getBatchId(); + Assert.assertNotEquals(id2, id3); + } + }); + } + + @Test + public void testConcurrentBatchIdUniqueness() throws Exception { + int threadCount = 8; + int buffersPerThread = 500; + int totalBuffers = threadCount * buffersPerThread; + Set batchIds = ConcurrentHashMap.newKeySet(); + CountDownLatch startLatch = new CountDownLatch(1); + CountDownLatch doneLatch = new CountDownLatch(threadCount); + + Thread[] threads = new Thread[threadCount]; + for (int t = 0; t < threadCount; t++) { + threads[t] = new Thread(() -> { + try { + startLatch.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + try { + for (int i = 0; i < buffersPerThread; i++) { + MicrobatchBuffer buf = new MicrobatchBuffer(64); + batchIds.add(buf.getBatchId()); + buf.close(); + } + } finally { + doneLatch.countDown(); + } + }); + threads[t].start(); + } + + startLatch.countDown(); + assertTrue("Threads did not finish in time", doneLatch.await(30, TimeUnit.SECONDS)); + + assertEquals( + "Duplicate batch IDs detected: expected " + totalBuffers + " unique IDs but got " + batchIds.size(), + totalBuffers, + batchIds.size() + ); + } + + @Test + public void testConcurrentResetBatchIdUniqueness() throws Exception { + int threadCount = 8; + int resetsPerThread = 500; + int totalIds = threadCount * resetsPerThread; + Set batchIds = ConcurrentHashMap.newKeySet(); + CountDownLatch startLatch = new CountDownLatch(1); + CountDownLatch doneLatch = new CountDownLatch(threadCount); + + Thread[] threads = new Thread[threadCount]; + for (int t = 0; t < threadCount; t++) { + threads[t] = new Thread(() -> { + try { + startLatch.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + try { + MicrobatchBuffer buf = new MicrobatchBuffer(64); + for (int i = 0; i < resetsPerThread; i++) { + buf.seal(); + buf.markSending(); + buf.markRecycled(); + buf.reset(); + batchIds.add(buf.getBatchId()); + } + buf.close(); + } finally { + doneLatch.countDown(); + } + }); + threads[t].start(); + } + + startLatch.countDown(); + assertTrue("Threads did not finish in time", doneLatch.await(30, TimeUnit.SECONDS)); + + assertEquals( + "Duplicate batch IDs from reset(): expected " + totalIds + " unique IDs but got " + batchIds.size(), + totalIds, + batchIds.size() + ); + } + + @Test + public void testConcurrentStateTransitions() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + AtomicReference error = new AtomicReference<>(); + CountDownLatch userDone = new CountDownLatch(1); + CountDownLatch ioDone = new CountDownLatch(1); + + // Simulate user thread + Thread userThread = new Thread(() -> { + try { + buffer.writeByte((byte) 1); + buffer.incrementRowCount(); + buffer.seal(); + userDone.countDown(); + + // Wait for I/O thread to recycle + buffer.awaitRecycled(); + + // Reset and write again + buffer.reset(); + buffer.writeByte((byte) 2); + } catch (Throwable t) { + error.set(t); + } + }); + + // Simulate I/O thread + Thread ioThread = new Thread(() -> { + try { + userDone.await(); + buffer.markSending(); + + // Simulate sending + Thread.sleep(10); + + buffer.markRecycled(); + ioDone.countDown(); + } catch (Throwable t) { + error.set(t); + } + }); + + userThread.start(); + ioThread.start(); + + userThread.join(1000); + ioThread.join(1000); + + Assert.assertNull(error.get()); + Assert.assertTrue(buffer.isFilling()); + Assert.assertEquals(1, buffer.getBufferPos()); + } + }); + } + + @Test + public void testConstructionWithCustomThresholds() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024, 100, 4096, 1_000_000_000L)) { + Assert.assertEquals(1024, buffer.getBufferCapacity()); + Assert.assertTrue(buffer.isFilling()); + } + }); + } + + @Test + public void testConstructionWithDefaultThresholds() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + Assert.assertEquals(1024, buffer.getBufferCapacity()); + Assert.assertEquals(0, buffer.getBufferPos()); + Assert.assertEquals(0, buffer.getRowCount()); + Assert.assertTrue(buffer.isFilling()); + Assert.assertFalse(buffer.hasData()); + } + }); + } + + @Test(expected = IllegalArgumentException.class) + public void testConstructionWithNegativeCapacity() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer ignored = new MicrobatchBuffer(-1)) { + Assert.fail("Should have thrown"); + } + }); + } + + @Test(expected = IllegalArgumentException.class) + public void testConstructionWithZeroCapacity() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer ignored = new MicrobatchBuffer(0)) { + Assert.fail("Should have thrown"); + } + }); + } + + @Test + public void testEnsureCapacityGrows() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.ensureCapacity(2000); + Assert.assertTrue(buffer.getBufferCapacity() >= 2000); + } + }); + } + + @Test + public void testEnsureCapacityNoGrowth() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.ensureCapacity(512); + Assert.assertEquals(1024, buffer.getBufferCapacity()); // No change + } + }); + } + + @Test + public void testFirstRowTimeIsRecorded() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + Assert.assertEquals(0, buffer.getAgeNanos()); + + buffer.incrementRowCount(); + long age1 = buffer.getAgeNanos(); + Assert.assertTrue(age1 >= 0); + + Thread.sleep(10); + + long age2 = buffer.getAgeNanos(); + Assert.assertTrue(age2 > age1); + } + }); + } + + @Test + public void testFullStateLifecycle() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + // FILLING + Assert.assertTrue(buffer.isFilling()); + buffer.writeByte((byte) 1); + buffer.incrementRowCount(); + + // FILLING -> SEALED + buffer.seal(); + Assert.assertTrue(buffer.isSealed()); + + // SEALED -> SENDING + buffer.markSending(); + Assert.assertTrue(buffer.isSending()); + + // SENDING -> RECYCLED + buffer.markRecycled(); + Assert.assertTrue(buffer.isRecycled()); + + // RECYCLED -> FILLING (reset) + buffer.reset(); + Assert.assertTrue(buffer.isFilling()); + Assert.assertFalse(buffer.hasData()); + } + }); + } + + @Test + public void testIncrementRowCount() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + Assert.assertEquals(0, buffer.getRowCount()); + buffer.incrementRowCount(); + Assert.assertEquals(1, buffer.getRowCount()); + buffer.incrementRowCount(); + Assert.assertEquals(2, buffer.getRowCount()); + } + }); + } + + @Test(expected = IllegalStateException.class) + public void testIncrementRowCountWhenSealed() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.seal(); + buffer.incrementRowCount(); // Should throw + } + }); + } + + @Test + public void testInitialState() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + Assert.assertEquals(MicrobatchBuffer.STATE_FILLING, buffer.getState()); + Assert.assertTrue(buffer.isFilling()); + Assert.assertFalse(buffer.isSealed()); + Assert.assertFalse(buffer.isSending()); + Assert.assertFalse(buffer.isRecycled()); + Assert.assertFalse(buffer.isInUse()); + } + }); + } + + @Test + public void testMarkRecycledTransition() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.seal(); + buffer.markSending(); + buffer.markRecycled(); + + Assert.assertEquals(MicrobatchBuffer.STATE_RECYCLED, buffer.getState()); + Assert.assertTrue(buffer.isRecycled()); + Assert.assertFalse(buffer.isInUse()); + } + }); + } + + @Test(expected = IllegalStateException.class) + public void testMarkRecycledWhenNotSending() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.seal(); + buffer.markRecycled(); // Should throw - not sending + } + }); + } + + @Test + public void testMarkSendingTransition() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.seal(); + buffer.markSending(); + + Assert.assertEquals(MicrobatchBuffer.STATE_SENDING, buffer.getState()); + Assert.assertTrue(buffer.isSending()); + Assert.assertTrue(buffer.isInUse()); + } + }); + } + + @Test(expected = IllegalStateException.class) + public void testMarkSendingWhenNotSealed() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.markSending(); // Should throw - not sealed + } + }); + } + + @Test + public void testResetFromRecycled() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.writeByte((byte) 1); + buffer.incrementRowCount(); + long oldBatchId = buffer.getBatchId(); + + buffer.seal(); + buffer.markSending(); + buffer.markRecycled(); + buffer.reset(); + + Assert.assertTrue(buffer.isFilling()); + Assert.assertEquals(0, buffer.getBufferPos()); + Assert.assertEquals(0, buffer.getRowCount()); + Assert.assertNotEquals(oldBatchId, buffer.getBatchId()); + } + }); + } + + @Test(expected = IllegalStateException.class) + public void testResetWhenSealed() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.seal(); + buffer.reset(); // Should throw + } + }); + } + + @Test(expected = IllegalStateException.class) + public void testResetWhenSending() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.seal(); + buffer.markSending(); + buffer.reset(); // Should throw + } + }); + } + + @Test + public void testRollbackSealForRetry() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.writeByte((byte) 1); + buffer.incrementRowCount(); + + buffer.seal(); + Assert.assertTrue(buffer.isSealed()); + + buffer.rollbackSealForRetry(); + Assert.assertTrue(buffer.isFilling()); + + // Verify the same batch remains writable after rollback. + buffer.writeByte((byte) 2); + buffer.incrementRowCount(); + Assert.assertEquals(2, buffer.getBufferPos()); + Assert.assertEquals(2, buffer.getRowCount()); + } + }); + } + + @Test(expected = IllegalStateException.class) + public void testRollbackSealWhenNotSealed() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.rollbackSealForRetry(); // Should throw - not sealed + } + }); + } + + @Test + public void testSealTransition() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.writeByte((byte) 1); + buffer.seal(); + + Assert.assertEquals(MicrobatchBuffer.STATE_SEALED, buffer.getState()); + Assert.assertFalse(buffer.isFilling()); + Assert.assertTrue(buffer.isSealed()); + Assert.assertTrue(buffer.isInUse()); + } + }); + } + + @Test(expected = IllegalStateException.class) + public void testSealWhenNotFilling() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.seal(); + buffer.seal(); // Should throw + } + }); + } + + @Test + public void testSetBufferPos() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.setBufferPos(100); + Assert.assertEquals(100, buffer.getBufferPos()); + } + }); + } + + @Test(expected = IllegalArgumentException.class) + public void testSetBufferPosNegative() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.setBufferPos(-1); + } + }); + } + + @Test(expected = IllegalArgumentException.class) + public void testSetBufferPosOutOfBounds() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.setBufferPos(2000); + } + }); + } + + @Test + public void testStateName() { + Assert.assertEquals("FILLING", MicrobatchBuffer.stateName(MicrobatchBuffer.STATE_FILLING)); + Assert.assertEquals("SEALED", MicrobatchBuffer.stateName(MicrobatchBuffer.STATE_SEALED)); + Assert.assertEquals("SENDING", MicrobatchBuffer.stateName(MicrobatchBuffer.STATE_SENDING)); + Assert.assertEquals("RECYCLED", MicrobatchBuffer.stateName(MicrobatchBuffer.STATE_RECYCLED)); + Assert.assertEquals("UNKNOWN(99)", MicrobatchBuffer.stateName(99)); + } + + @Test + public void testToString() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.writeByte((byte) 1); + buffer.incrementRowCount(); + + String str = buffer.toString(); + Assert.assertTrue(str.contains("MicrobatchBuffer")); + Assert.assertTrue(str.contains("state=FILLING")); + Assert.assertTrue(str.contains("rows=1")); + Assert.assertTrue(str.contains("bytes=1")); + } + }); + } + + @Test + public void testWriteBeyondInitialCapacity() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(16)) { + // Write more than initial capacity + for (int i = 0; i < 100; i++) { + buffer.writeByte((byte) i); + } + Assert.assertEquals(100, buffer.getBufferPos()); + Assert.assertTrue(buffer.getBufferCapacity() >= 100); + + // Verify data integrity after growth + for (int i = 0; i < 100; i++) { + byte read = Unsafe.getUnsafe().getByte(buffer.getBufferPtr() + i); + Assert.assertEquals((byte) i, read); + } + } + }); + } + + @Test + public void testWriteByte() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.writeByte((byte) 0x42); + Assert.assertEquals(1, buffer.getBufferPos()); + Assert.assertTrue(buffer.hasData()); + + byte read = Unsafe.getUnsafe().getByte(buffer.getBufferPtr()); + Assert.assertEquals((byte) 0x42, read); + } + }); + } + + @Test + public void testWriteFromNativeMemory() throws Exception { + assertMemoryLeak(() -> { + long src = Unsafe.malloc(10, MemoryTag.NATIVE_DEFAULT); + try { + // Fill source with test data + for (int i = 0; i < 10; i++) { + Unsafe.getUnsafe().putByte(src + i, (byte) (i + 100)); + } + + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.write(src, 10); + Assert.assertEquals(10, buffer.getBufferPos()); + + // Verify data + for (int i = 0; i < 10; i++) { + byte read = Unsafe.getUnsafe().getByte(buffer.getBufferPtr() + i); + Assert.assertEquals((byte) (i + 100), read); + } + } + } finally { + Unsafe.free(src, 10, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testWriteMultipleBytes() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + for (int i = 0; i < 100; i++) { + buffer.writeByte((byte) i); + } + Assert.assertEquals(100, buffer.getBufferPos()); + + // Verify data + for (int i = 0; i < 100; i++) { + byte read = Unsafe.getUnsafe().getByte(buffer.getBufferPtr() + i); + Assert.assertEquals((byte) i, read); + } + } + }); + } + + @Test(expected = IllegalStateException.class) + public void testWriteWhenSealed() throws Exception { + assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.seal(); + buffer.writeByte((byte) 1); // Should throw + } + }); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java new file mode 100644 index 0000000..75ce11c --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java @@ -0,0 +1,539 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; +import static org.junit.Assert.*; + +public class NativeBufferWriterTest { + + @Test + public void testEnsureCapacityGrowsBuffer() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter(16)) { + assertEquals(16, writer.getCapacity()); + writer.ensureCapacity(32); + assertTrue(writer.getCapacity() >= 32); + } + }); + } + + @Test + public void testGrowBuffer() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter(16)) { + // Write more than initial capacity + for (int i = 0; i < 100; i++) { + writer.putLong(i); + } + Assert.assertEquals(800, writer.getPosition()); + // Verify data + for (int i = 0; i < 100; i++) { + Assert.assertEquals(i, Unsafe.getUnsafe().getLong(writer.getBufferPtr() + i * 8)); + } + } + }); + } + + @Test + public void testMultipleWrites() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putByte((byte) 'Q'); + writer.putByte((byte) 'W'); + writer.putByte((byte) 'P'); + writer.putByte((byte) '1'); + writer.putByte((byte) 1); // Version + writer.putByte((byte) 0); // Flags + writer.putShort((short) 1); // Table count + writer.putInt(0); // Payload length placeholder + + Assert.assertEquals(12, writer.getPosition()); + + // Verify QWP1 header + Assert.assertEquals((byte) 'Q', Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 'W', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + Assert.assertEquals((byte) 'P', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 2)); + Assert.assertEquals((byte) '1', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 3)); + } + }); + } + + @Test + public void testNativeBufferWriterUtf8LengthInvalidSurrogatePair() throws Exception { + assertMemoryLeak(() -> { + // High surrogate followed by non-low-surrogate: '?' (1) + 'X' (1) = 2 + assertEquals(2, NativeBufferWriter.utf8Length("\uD800X")); + // Lone high surrogate at end: '?' (1) + assertEquals(1, NativeBufferWriter.utf8Length("\uD800")); + // Lone low surrogate: '?' (1) + assertEquals(1, NativeBufferWriter.utf8Length("\uDC00")); + // Valid pair still works: 4 bytes + assertEquals(4, NativeBufferWriter.utf8Length("\uD83D\uDE00")); + }); + } + + @Test + public void testPatchInt() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putInt(0); // Placeholder at offset 0 + writer.putInt(100); // At offset 4 + writer.patchInt(0, 42); // Patch first int + Assert.assertEquals(42, Unsafe.getUnsafe().getInt(writer.getBufferPtr())); + Assert.assertEquals(100, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 4)); + } + }); + } + + @Test + public void testPatchIntAtLastValidOffset() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter(16)) { + writer.putLong(0L); // 8 bytes, position = 8 + // Patch at offset 4 covers bytes [4..7], exactly at the boundary + writer.patchInt(4, 0x1234); + assertEquals(0x1234, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 4)); + } + }); + } + + @Test + public void testPatchIntAtValidOffset() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter(16)) { + writer.putInt(0); // placeholder at offset 0 + writer.putInt(0xBEEF); // data at offset 4 + // Patch the placeholder + writer.patchInt(0, 0xCAFE); + assertEquals(0xCAFE, Unsafe.getUnsafe().getInt(writer.getBufferPtr())); + assertEquals(0xBEEF, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 4)); + } + }); + } + + @Test + public void testPutBlockOfBytes() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter(); + NativeBufferWriter source = new NativeBufferWriter()) { + // Prepare source data + source.putByte((byte) 1); + source.putByte((byte) 2); + source.putByte((byte) 3); + source.putByte((byte) 4); + + // Copy to writer + writer.putBlockOfBytes(source.getBufferPtr(), 4); + Assert.assertEquals(4, writer.getPosition()); + Assert.assertEquals((byte) 1, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 2, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + Assert.assertEquals((byte) 3, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 2)); + Assert.assertEquals((byte) 4, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 3)); + } + }); + } + + @Test + public void testPutBlockOfBytesRejectsLenExceedingIntMax() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter(16)) { + try { + writer.putBlockOfBytes(0, (long) Integer.MAX_VALUE + 1); + fail("expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains("len")); + } + } + }); + } + + @Test + public void testPutUtf8InvalidSurrogatePair() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter(64)) { + // High surrogate \uD800 followed by non-low-surrogate 'X'. + // Should produce '?' for the lone high surrogate, then 'X'. + writer.putUtf8("\uD800X"); + assertEquals(2, writer.getPosition()); + assertEquals((byte) '?', Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + assertEquals((byte) 'X', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + } + }); + } + + @Test + public void testPutUtf8LoneHighSurrogateAtEnd() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter(64)) { + writer.putUtf8("\uD800"); + assertEquals(1, writer.getPosition()); + assertEquals((byte) '?', Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + } + }); + } + + @Test + public void testPutUtf8LoneLowSurrogate() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter(64)) { + writer.putUtf8("\uDC00"); + assertEquals(1, writer.getPosition()); + assertEquals((byte) '?', Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + } + }); + } + + @Test + public void testPutUtf8LoneSurrogateMatchesUtf8Length() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter(64)) { + // Verify putUtf8 and utf8Length agree for all lone surrogate cases + String[] cases = {"\uD800", "\uDBFF", "\uDC00", "\uDFFF", "\uD800X", "A\uDC00B"}; + for (String s : cases) { + writer.reset(); + writer.putUtf8(s); + assertEquals("length mismatch for: " + s.codePoints() + .mapToObj(cp -> String.format("U+%04X", cp)) + .reduce((a, b) -> a + " " + b).orElse(""), + NativeBufferWriter.utf8Length(s), writer.getPosition()); + } + } + }); + } + + @Test + public void testQwpBufferWriterUtf8LengthInvalidSurrogatePair() throws Exception { + assertMemoryLeak(() -> { + // High surrogate followed by non-low-surrogate: '?' (1) + 'X' (1) = 2 + assertEquals(2, NativeBufferWriter.utf8Length("\uD800X")); + // Lone high surrogate at end: '?' (1) + assertEquals(1, NativeBufferWriter.utf8Length("\uD800")); + // Lone low surrogate: '?' (1) + assertEquals(1, NativeBufferWriter.utf8Length("\uDC00")); + // Valid pair still works: 4 bytes + assertEquals(4, NativeBufferWriter.utf8Length("\uD83D\uDE00")); + }); + } + + @Test + public void testReset() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putInt(12345); + Assert.assertEquals(4, writer.getPosition()); + writer.reset(); + Assert.assertEquals(0, writer.getPosition()); + // Can write again + writer.putByte((byte) 0xFF); + Assert.assertEquals(1, writer.getPosition()); + } + }); + } + + @Test + public void testSkipAdvancesPosition() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter(16)) { + writer.skip(4); + assertEquals(4, writer.getPosition()); + writer.skip(8); + assertEquals(12, writer.getPosition()); + } + }); + } + + @Test + public void testSkipBeyondCapacityGrowsBuffer() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter(16)) { + // skip past the 16-byte buffer — must grow, not corrupt memory + writer.skip(32); + assertEquals(32, writer.getPosition()); + assertTrue(writer.getCapacity() >= 32); + // writing after the skip must also succeed + writer.putInt(0xCAFE); + assertEquals(36, writer.getPosition()); + assertEquals(0xCAFE, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 32)); + } + }); + } + + @Test + public void testSkipThenPatchInt() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter(8)) { + int patchOffset = writer.getPosition(); + writer.skip(4); // reserve space for a length field + writer.putInt(0xDEAD); + // Patch the reserved space + writer.patchInt(patchOffset, 4); + assertEquals(0x4, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + patchOffset)); + assertEquals(0xDEAD, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 4)); + } + }); + } + + @Test + public void testUtf8Length() throws Exception { + assertMemoryLeak(() -> { + Assert.assertEquals(0, NativeBufferWriter.utf8Length(null)); + Assert.assertEquals(0, NativeBufferWriter.utf8Length("")); + Assert.assertEquals(5, NativeBufferWriter.utf8Length("hello")); + Assert.assertEquals(2, NativeBufferWriter.utf8Length("ñ")); + Assert.assertEquals(3, NativeBufferWriter.utf8Length("€")); + }); + } + + @Test + public void testWriteByte() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putByte((byte) 0x42); + Assert.assertEquals(1, writer.getPosition()); + Assert.assertEquals((byte) 0x42, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + } + }); + } + + @Test + public void testWriteDouble() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putDouble(3.14159265359); + Assert.assertEquals(8, writer.getPosition()); + Assert.assertEquals(3.14159265359, Unsafe.getUnsafe().getDouble(writer.getBufferPtr()), 0.0000000001); + } + }); + } + + @Test + public void testWriteEmptyString() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putString(""); + Assert.assertEquals(1, writer.getPosition()); + Assert.assertEquals((byte) 0, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + } + }); + } + + @Test + public void testWriteFloat() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putFloat(3.14f); + Assert.assertEquals(4, writer.getPosition()); + Assert.assertEquals(3.14f, Unsafe.getUnsafe().getFloat(writer.getBufferPtr()), 0.0001f); + } + }); + } + + @Test + public void testWriteInt() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putInt(0x12345678); + Assert.assertEquals(4, writer.getPosition()); + // Little-endian + Assert.assertEquals(0x12345678, Unsafe.getUnsafe().getInt(writer.getBufferPtr())); + } + }); + } + + @Test + public void testWriteLong() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putLong(0x123456789ABCDEF0L); + Assert.assertEquals(8, writer.getPosition()); + Assert.assertEquals(0x123456789ABCDEF0L, Unsafe.getUnsafe().getLong(writer.getBufferPtr())); + } + }); + } + + @Test + public void testWriteLongBigEndian() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putLongBE(0x0102030405060708L); + Assert.assertEquals(8, writer.getPosition()); + // Check big-endian byte order + long ptr = writer.getBufferPtr(); + Assert.assertEquals((byte) 0x01, Unsafe.getUnsafe().getByte(ptr)); + Assert.assertEquals((byte) 0x02, Unsafe.getUnsafe().getByte(ptr + 1)); + Assert.assertEquals((byte) 0x03, Unsafe.getUnsafe().getByte(ptr + 2)); + Assert.assertEquals((byte) 0x04, Unsafe.getUnsafe().getByte(ptr + 3)); + Assert.assertEquals((byte) 0x05, Unsafe.getUnsafe().getByte(ptr + 4)); + Assert.assertEquals((byte) 0x06, Unsafe.getUnsafe().getByte(ptr + 5)); + Assert.assertEquals((byte) 0x07, Unsafe.getUnsafe().getByte(ptr + 6)); + Assert.assertEquals((byte) 0x08, Unsafe.getUnsafe().getByte(ptr + 7)); + } + }); + } + + @Test + public void testWriteNullString() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putString(null); + Assert.assertEquals(1, writer.getPosition()); + Assert.assertEquals((byte) 0, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + } + }); + } + + @Test + public void testWriteShort() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putShort((short) 0x1234); + Assert.assertEquals(2, writer.getPosition()); + // Little-endian + Assert.assertEquals((byte) 0x34, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 0x12, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + } + }); + } + + @Test + public void testWriteString() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putString("hello"); + // Length (1 byte varint) + 5 bytes + Assert.assertEquals(6, writer.getPosition()); + // Check length + Assert.assertEquals((byte) 5, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + // Check content + Assert.assertEquals((byte) 'h', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + Assert.assertEquals((byte) 'e', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 2)); + Assert.assertEquals((byte) 'l', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 3)); + Assert.assertEquals((byte) 'l', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 4)); + Assert.assertEquals((byte) 'o', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 5)); + } + }); + } + + @Test + public void testWriteUtf8Ascii() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putUtf8("ABC"); + Assert.assertEquals(3, writer.getPosition()); + Assert.assertEquals((byte) 'A', Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 'B', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + Assert.assertEquals((byte) 'C', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 2)); + } + }); + } + + @Test + public void testWriteUtf8ThreeByte() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + // € is 3 bytes in UTF-8 + writer.putUtf8("€"); + Assert.assertEquals(3, writer.getPosition()); + Assert.assertEquals((byte) 0xE2, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 0x82, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + Assert.assertEquals((byte) 0xAC, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 2)); + } + }); + } + + @Test + public void testWriteUtf8TwoByte() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + // ñ is 2 bytes in UTF-8 + writer.putUtf8("ñ"); + Assert.assertEquals(2, writer.getPosition()); + Assert.assertEquals((byte) 0xC3, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 0xB1, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + } + }); + } + + @Test + public void testWriteVarintLarge() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + // Test larger value + writer.putVarint(16384); + Assert.assertEquals(3, writer.getPosition()); + // LEB128: 16384 = 0x80 0x80 0x01 + Assert.assertEquals((byte) 0x80, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 0x80, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + Assert.assertEquals((byte) 0x01, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 2)); + } + }); + } + + @Test + public void testVarintSizeMatchesEncodedLength() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + long[] values = {0, 1, 127, 128, 16383, 16384, 2_097_151, 2_097_152, Long.MAX_VALUE}; + for (long value : values) { + writer.reset(); + writer.putVarint(value); + Assert.assertEquals("value=" + value, NativeBufferWriter.varintSize(value), writer.getPosition()); + } + } + }); + } + + @Test + public void testWriteVarintMedium() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + // Two bytes for 128 + writer.putVarint(128); + Assert.assertEquals(2, writer.getPosition()); + // LEB128: 128 = 0x80 0x01 + Assert.assertEquals((byte) 0x80, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 0x01, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + } + }); + } + + @Test + public void testWriteVarintSmall() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + // Single byte for values < 128 + writer.putVarint(127); + Assert.assertEquals(1, writer.getPosition()); + Assert.assertEquals((byte) 127, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + } + }); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpDeltaDictRollbackTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpDeltaDictRollbackTest.java new file mode 100644 index 0000000..eeccce0 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpDeltaDictRollbackTest.java @@ -0,0 +1,94 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.InFlightWindow; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.test.AbstractTest; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.time.temporal.ChronoUnit; + +/** + * Verifies that maxSentSymbolId and sentSchemaHashes are not updated + * when the send fails, so the next batch's delta dictionary correctly + * re-includes symbols the server never received. + */ +public class QwpDeltaDictRollbackTest extends AbstractTest { + + @Test + public void testSyncFlushFailureDoesNotAdvanceMaxSentSymbolId() throws Exception { + assertMemoryLeak(() -> { + // Sync mode (window=1), not connected to any server + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 0, 1); + try { + // Bypass ensureConnected() by marking as connected. + // Leave client null so sendBinary() will throw. + setField(sender, "connected", true); + setField(sender, "inFlightWindow", new InFlightWindow(1, InFlightWindow.DEFAULT_TIMEOUT_MS)); + + // Buffer a row with a symbol — this registers symbol id 0 + // in the global dictionary and sets currentBatchMaxSymbolId = 0 + sender.table("t") + .symbol("s", "val1") + .at(1, ChronoUnit.MICROS); + + // maxSentSymbolId should still be -1 (nothing sent yet) + Assert.assertEquals(-1, sender.getMaxSentSymbolId()); + + // flush() -> flushSync() -> encode succeeds -> client.sendBinary() throws NPE + // because client is null (we never actually connected) + try { + sender.flush(); + Assert.fail("Expected LineSenderException from null client"); + } catch (LineSenderException expected) { + // sendBinary() on null client, wrapped by flushSync() + } + + // The fix: maxSentSymbolId must remain -1 because the send failed. + // Without the fix, it would have been advanced to 0 before the throw, + // causing the next batch's delta dictionary to omit symbol "val1". + Assert.assertEquals( + "maxSentSymbolId must not advance when send fails", + -1, sender.getMaxSentSymbolId() + ); + } finally { + // Mark as not connected so close() doesn't try to flush again + setField(sender, "connected", false); + sender.close(); + } + }); + } + + private static void setField(Object target, String fieldName, Object value) throws Exception { + Field f = target.getClass().getDeclaredField(fieldName); + f.setAccessible(true); + f.set(target, value); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java new file mode 100644 index 0000000..d675401 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java @@ -0,0 +1,2597 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.line.array.DoubleArray; +import io.questdb.client.cutlass.line.array.LongArray; +import io.questdb.client.cutlass.qwp.client.QwpUdpSender; +import io.questdb.client.cutlass.qwp.protocol.QwpColumnDef; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.network.NetworkFacadeImpl; +import io.questdb.client.std.Decimal128; +import io.questdb.client.std.Decimal256; +import io.questdb.client.std.Decimal64; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpUdpSenderTest { + + @Test + public void testAdaptiveHeadroomFlushesCommittedRowsBeforeNextRowStarts() throws Exception { + assertMemoryLeak(() -> { + String alpha = repeat('a', 256); + String beta = repeat('b', 256); + String gamma = repeat('c', 256); + List rows = Arrays.asList( + row("t", sender -> sender.table("t") + .longColumn("x", 1) + .stringColumn("s", alpha) + .atNow(), + "x", 1L, + "s", alpha), + row("t", sender -> sender.table("t") + .longColumn("x", 2) + .stringColumn("s", beta) + .atNow(), + "x", 2L, + "s", beta), + row("t", sender -> sender.table("t") + .longColumn("x", 3) + .stringColumn("s", gamma) + .atNow(), + "x", 3L, + "s", gamma) + ); + + int oneRowPacket = fullPacketSize(rows.subList(0, 1)); + int twoRowPacket = fullPacketSize(rows.subList(0, 2)); + int maxDatagramSize = oneRowPacket + 16; + Assert.assertTrue("expected overflow boundary between rows 1 and 2", maxDatagramSize < twoRowPacket); + + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) { + sender.table("t") + .longColumn("x", 1) + .stringColumn("s", alpha) + .atNow(); + Assert.assertEquals(0, nf.sendCount); + + sender.longColumn("x", 2) + .stringColumn("s", beta) + .atNow(); + Assert.assertEquals("expected adaptive post-commit flush for row 2", 2, nf.sendCount); + + sender.longColumn("x", 3) + .stringColumn("s", gamma) + .atNow(); + Assert.assertEquals("expected row 3 to start on a fresh datagram", 3, nf.sendCount); + + sender.flush(); + } + + RunResult result = new RunResult(nf.packets, nf.lengths, nf.sendCount); + Assert.assertEquals(3, result.sendCount); + assertPacketsWithinLimit(result, maxDatagramSize); + assertRowsEqual(expectedRows(rows), decodeRows(nf.packets)); + }); + } + + @Test + public void testAdaptiveHeadroomStateIsPerTable() throws Exception { + assertMemoryLeak(() -> { + String alpha = repeat('a', 256); + String beta = repeat('b', 256); + List bigRows = Arrays.asList( + row("big", sender -> sender.table("big") + .longColumn("x", 1) + .stringColumn("s", alpha) + .atNow(), + "x", 1L, + "s", alpha), + row("big", sender -> sender.table("big") + .longColumn("x", 2) + .stringColumn("s", beta) + .atNow(), + "x", 2L, + "s", beta) + ); + List smallRows = Arrays.asList( + row("small", sender -> sender.table("small") + .longColumn("x", 10) + .atNow(), + "x", 10L), + row("small", sender -> sender.table("small") + .longColumn("x", 11) + .atNow(), + "x", 11L) + ); + + int oneBigRowPacket = fullPacketSize(bigRows.subList(0, 1)); + int twoBigRowPacket = fullPacketSize(bigRows); + int twoSmallRowPacket = fullPacketSize(smallRows); + int maxDatagramSize = oneBigRowPacket + 16; + Assert.assertTrue("expected overflow boundary between big rows", maxDatagramSize < twoBigRowPacket); + Assert.assertTrue("expected small rows to fit together", twoSmallRowPacket <= maxDatagramSize); + + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) { + sender.table("big") + .longColumn("x", 1) + .stringColumn("s", alpha) + .atNow(); + sender.longColumn("x", 2) + .stringColumn("s", beta) + .atNow(); + Assert.assertEquals(2, nf.sendCount); + + sender.table("small") + .longColumn("x", 10) + .atNow(); + Assert.assertEquals("big-table headroom must not flush small-table row 1", 2, nf.sendCount); + + sender.longColumn("x", 11) + .atNow(); + Assert.assertEquals("small-table rows should share a datagram", 2, nf.sendCount); + + sender.flush(); + } + + RunResult result = new RunResult(nf.packets, nf.lengths, nf.sendCount); + Assert.assertEquals(3, result.sendCount); + assertPacketsWithinLimit(result, maxDatagramSize); + assertRowsEqual(Arrays.asList( + decodedRow("big", "x", 1L, "s", alpha), + decodedRow("big", "x", 2L, "s", beta), + decodedRow("small", "x", 10L), + decodedRow("small", "x", 11L) + ), decodeRows(nf.packets)); + }); + } + + @Test + public void testArrayWrapperStagingSnapshotsMutationAndCancelRow() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024); + DoubleArray doubleArray = new DoubleArray(2)) { + sender.table("arrays"); + long[] longValues = {1, 2}; + + doubleArray.append(1.5).append(2.5); + sender.longArray("la", longValues); + sender.doubleArray("da", doubleArray); + + longValues[0] = 9; + longValues[1] = 10; + doubleArray.clear(); + doubleArray.append(9.5).append(10.5); + sender.cancelRow(); + + sender.longArray("la", longValues); + sender.doubleArray("da", doubleArray); + longValues[0] = 100; + longValues[1] = 200; + doubleArray.clear(); + doubleArray.append(100.5).append(200.5); + sender.atNow(); + sender.flush(); + } + + assertRowsEqual( + Arrays.asList(decodedRow( + "arrays", + "la", longArrayValue(shape(2), 9, 10), + "da", doubleArrayValue(shape(2), 9.5, 10.5) + )), + decodeRows(nf.packets) + ); + }); + } + + @Test + public void testAtMicrosOversizeFailureRollsBackWithoutLeakingTimestampState() throws Exception { + assertMemoryLeak(() -> { + String large = repeat('x', 5000); + List oversizedRow = Arrays.asList( + row("t", sender -> sender.table("t") + .longColumn("a", 2) + .stringColumn("s", large) + .at(2_000_000L, ChronoUnit.MICROS), + "a", 2L, + "s", large, + "", 2_000_000L) + ); + int maxDatagramSize = fullPacketSize(oversizedRow) - 1; + + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) { + sender.table("t") + .longColumn("a", 1) + .at(1_000_000L, ChronoUnit.MICROS); + + assertThrowsContains("single row exceeds maximum datagram size", () -> + sender.longColumn("a", 2) + .stringColumn("s", large) + .at(2_000_000L, ChronoUnit.MICROS) + ); + Assert.assertEquals(1, nf.sendCount); + + sender.longColumn("a", 3).at(3_000_000L, ChronoUnit.MICROS); + sender.flush(); + } + + Assert.assertEquals(2, nf.sendCount); + assertRowsEqual(Arrays.asList( + decodedRow("t", "a", 1L, "", 1_000_000L), + decodedRow("t", "a", 3L, "", 3_000_000L) + ), decodeRows(nf.packets)); + }); + } + + @Test + public void testAtNanosOversizeFailureRollsBackWithoutLeakingTimestampState() throws Exception { + assertMemoryLeak(() -> { + String large = repeat('x', 5000); + List oversizedRow = Arrays.asList( + row("tn", sender -> sender.table("tn") + .longColumn("a", 2) + .stringColumn("s", large) + .at(2_000_000L, ChronoUnit.NANOS), + "a", 2L, + "s", large, + "", 2_000_000L) + ); + int maxDatagramSize = fullPacketSize(oversizedRow) - 1; + + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) { + sender.table("tn") + .longColumn("a", 1) + .at(1_000_000L, ChronoUnit.NANOS); + + assertThrowsContains("single row exceeds maximum datagram size", () -> + sender.longColumn("a", 2) + .stringColumn("s", large) + .at(2_000_000L, ChronoUnit.NANOS) + ); + Assert.assertEquals(1, nf.sendCount); + + sender.longColumn("a", 3).at(3_000_000L, ChronoUnit.NANOS); + sender.flush(); + } + + Assert.assertEquals(2, nf.sendCount); + assertRowsEqual(Arrays.asList( + decodedRow("tn", "a", 1L, "", 1_000_000L), + decodedRow("tn", "a", 3L, "", 3_000_000L) + ), decodeRows(nf.packets)); + }); + } + + @Test + public void testAtNowOversizeFailureRollsBackWithoutExplicitCancel() throws Exception { + assertMemoryLeak(() -> { + String large = repeat('x', 5000); + List oversizedRow = Arrays.asList( + row("t", sender -> sender.table("t") + .longColumn("a", 2) + .stringColumn("s", large) + .atNow(), + "a", 2L, + "s", large) + ); + int maxDatagramSize = fullPacketSize(oversizedRow) - 1; + + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) { + sender.table("t") + .longColumn("a", 1) + .atNow(); + + assertThrowsContains("single row exceeds maximum datagram size", () -> + sender.longColumn("a", 2) + .stringColumn("s", large) + .atNow() + ); + Assert.assertEquals(1, nf.sendCount); + + sender.longColumn("a", 3).atNow(); + sender.flush(); + } + + Assert.assertEquals(2, nf.sendCount); + assertRowsEqual(Arrays.asList( + decodedRow("t", "a", 1L), + decodedRow("t", "a", 3L) + ), decodeRows(nf.packets)); + }); + } + + @Test + public void testBoundedSenderArrayReplayPreservesRowsAndPacketLimit() throws Exception { + assertMemoryLeak(() -> { + long[] longValues = new long[128]; + double[] doubleValues = new double[128]; + for (int i = 0; i < 128; i++) { + longValues[i] = i * 3L; + doubleValues[i] = i * 1.25; + } + + long[][] longMatrix = new long[8][8]; + double[][] doubleMatrix = new double[8][8]; + for (int i = 0; i < 8; i++) { + for (int j = 0; j < 8; j++) { + longMatrix[i][j] = i * 100L + j; + doubleMatrix[i][j] = i * 10.0 + j + 0.5; + } + } + + List rows = Arrays.asList( + row("arrays", sender -> sender.table("arrays") + .symbol("sym", "alpha") + .longColumn("x", 1) + .longArray("la", longValues) + .doubleArray("da", doubleValues) + .atNow(), + "sym", "alpha", + "x", 1L, + "la", longArrayValue(shape(128), longValues), + "da", doubleArrayValue(shape(128), doubleValues)), + row("arrays", sender -> sender.table("arrays") + .symbol("sym", "beta") + .longColumn("x", 2) + .longArray("la", longMatrix) + .doubleArray("da", doubleMatrix) + .atNow(), + "sym", "beta", + "x", 2L, + "la", longArrayValue(shape(8, 8), flatten(longMatrix)), + "da", doubleArrayValue(shape(8, 8), flatten(doubleMatrix))), + row("arrays", sender -> sender.table("arrays") + .symbol("sym", "gamma") + .longColumn("x", 3) + .longArray("la", (long[]) null) + .doubleArray("da", (double[]) null) + .atNow(), + "sym", "gamma", + "x", 3L, + "la", null, + "da", null) + ); + + int maxDatagramSize = fullPacketSize(rows) - 1; + RunResult result = runScenario(rows, maxDatagramSize); + + Assert.assertTrue("expected at least one auto-flush", result.packets.size() > 1); + assertPacketsWithinLimit(result, maxDatagramSize); + assertRowsEqual(expectedRows(rows), decodeRows(result.packets)); + }); + } + + @Test + public void testBoundedSenderHigherDimensionalDoubleArrayWrapperPreservesRowsAndPacketLimit() throws Exception { + assertMemoryLeak(() -> { + List rows = Arrays.asList( + row("arrays", sender -> { + try (DoubleArray doubleArray = new DoubleArray(2, 1, 1, 2)) { + doubleArray.append(1.25).append(2.25).append(3.25).append(4.25); + sender.table("arrays") + .symbol("sym", "alpha") + .longArray("la", new long[][][]{{{1, 2}}, {{3, 4}}}) + .doubleArray("da", doubleArray) + .atNow(); + } + }, + "sym", "alpha", + "la", longArrayValue(shape(2, 1, 2), 1, 2, 3, 4), + "da", doubleArrayValue(shape(2, 1, 1, 2), 1.25, 2.25, 3.25, 4.25)), + row("arrays", sender -> { + try (DoubleArray doubleArray = new DoubleArray(1, 2, 1, 2)) { + doubleArray.append(10.5).append(20.5).append(30.5).append(40.5); + sender.table("arrays") + .symbol("sym", "beta") + .longArray("la", new long[][]{{10, 20}, {30, 40}}) + .doubleArray("da", doubleArray) + .atNow(); + } + }, + "sym", "beta", + "la", longArrayValue(shape(2, 2), 10, 20, 30, 40), + "da", doubleArrayValue(shape(1, 2, 1, 2), 10.5, 20.5, 30.5, 40.5)) + ); + + int maxDatagramSize = fullPacketSize(rows) - 1; + RunResult result = runScenario(rows, maxDatagramSize); + + Assert.assertTrue("expected at least one auto-flush", result.packets.size() > 1); + assertPacketsWithinLimit(result, maxDatagramSize); + assertRowsEqual(expectedRows(rows), decodeRows(result.packets)); + }); + } + + @Test + public void testBoundedSenderMixedNullablePaddingPreservesRowsAndPacketLimit() throws Exception { + assertMemoryLeak(() -> { + String alpha = repeat('a', 256); + String omega = repeat('z', 256); + List rows = Arrays.asList( + row("t", sender -> sender.table("t") + .symbol("sym", "v1") + .longColumn("x", 1) + .stringColumn("s", alpha) + .atNow(), + "sym", "v1", + "x", 1L, + "s", alpha), + row("t", sender -> sender.table("t") + .symbol("sym", null) + .longColumn("x", 2) + .atNow(), + "sym", null, + "x", 2L, + "s", null), + row("t", sender -> sender.table("t") + .symbol("sym", null) + .longColumn("x", 3) + .stringColumn("s", null) + .atNow(), + "sym", null, + "x", 3L, + "s", null), + row("t", sender -> sender.table("t") + .symbol("sym", "v2") + .longColumn("x", 4) + .stringColumn("s", omega) + .atNow(), + "sym", "v2", + "x", 4L, + "s", omega) + ); + + int maxDatagramSize = fullPacketSize(rows) - 1; + RunResult result = runScenario(rows, maxDatagramSize); + + Assert.assertTrue("expected at least one auto-flush", result.packets.size() > 1); + assertPacketsWithinLimit(result, maxDatagramSize); + assertRowsEqual(expectedRows(rows), decodeRows(result.packets)); + }); + } + + @Test + public void testBoundedSenderMixedTypesPreservesRowsAndPacketLimit() throws Exception { + assertMemoryLeak(() -> { + String msg1 = repeat('m', 240); + String msg2 = repeat('n', 240); + List rows = Arrays.asList( + row("mix", sender -> sender.table("mix") + .symbol("sym", "alpha") + .boolColumn("active", true) + .longColumn("count", 1) + .doubleColumn("value", 1.5) + .stringColumn("msg", msg1) + .decimalColumn("d64", Decimal64.fromLong(12345L, 2)) + .decimalColumn("d128", Decimal128.fromLong(678901234L, 4)) + .decimalColumn("d256", Decimal256.fromLong(9876543210L, 3)) + .timestampColumn("eventMicros", 123456L, ChronoUnit.MICROS) + .timestampColumn("eventNanos", 999L, ChronoUnit.NANOS) + .at(1_000_000L, ChronoUnit.MICROS), + "sym", "alpha", + "active", true, + "count", 1L, + "value", 1.5, + "msg", msg1, + "d64", decimal(12345L, 2), + "d128", decimal(678901234L, 4), + "d256", decimal(9876543210L, 3), + "eventMicros", 123456L, + "eventNanos", 999L, + "", 1_000_000L), + row("mix", sender -> sender.table("mix") + .symbol("sym", "beta") + .boolColumn("active", false) + .longColumn("count", 2) + .doubleColumn("value", 2.5) + .stringColumn("msg", msg2) + .decimalColumn("d64", Decimal64.fromLong(-67890L, 2)) + .decimalColumn("d128", Decimal128.fromLong(2222333344L, 4)) + .decimalColumn("d256", Decimal256.fromLong(7777777770L, 3)) + .timestampColumn("eventMicros", 654321L, ChronoUnit.MICROS) + .timestampColumn("eventNanos", 12345L, ChronoUnit.NANOS) + .at(2_000_000L, ChronoUnit.MICROS), + "sym", "beta", + "active", false, + "count", 2L, + "value", 2.5, + "msg", msg2, + "d64", decimal(-67890L, 2), + "d128", decimal(2222333344L, 4), + "d256", decimal(7777777770L, 3), + "eventMicros", 654321L, + "eventNanos", 12345L, + "", 2_000_000L) + ); + + int maxDatagramSize = fullPacketSize(rows) - 1; + RunResult result = runScenario(rows, maxDatagramSize); + + Assert.assertTrue("expected at least one auto-flush", result.packets.size() > 1); + assertPacketsWithinLimit(result, maxDatagramSize); + assertRowsEqual(expectedRows(rows), decodeRows(result.packets)); + }); + } + + @Test + public void testBoundedSenderNullableStringNullAcrossOverflowBoundaryPreservesRowsAndPacketLimit() throws Exception { + assertMemoryLeak(() -> { + String alpha = repeat('a', 512); + String marker1 = repeat('m', 64); + String marker2 = repeat('n', 64); + String marker3 = repeat('o', 64); + String omega = repeat('z', 128); + List rows = Arrays.asList( + row("t", sender -> sender.table("t") + .longColumn("x", 1) + .stringColumn("s", alpha) + .stringColumn("m", marker1) + .atNow(), + "x", 1L, + "s", alpha, + "m", marker1), + row("t", sender -> sender.table("t") + .longColumn("x", 2) + .stringColumn("s", null) + .stringColumn("m", marker2) + .atNow(), + "x", 2L, + "s", null, + "m", marker2), + row("t", sender -> sender.table("t") + .longColumn("x", 3) + .stringColumn("s", omega) + .stringColumn("m", marker3) + .atNow(), + "x", 3L, + "s", omega, + "m", marker3) + ); + int firstRowPacket = fullPacketSize(rows.subList(0, 1)); + int twoRowPacket = fullPacketSize(rows.subList(0, 2)); + int maxDatagramSize = firstRowPacket + 16; + Assert.assertTrue("expected overflow boundary between row 1 and row 2", maxDatagramSize < twoRowPacket); + + RunResult result = runScenario(rows, maxDatagramSize); + + Assert.assertTrue("expected at least one auto-flush", result.packets.size() > 1); + assertPacketsWithinLimit(result, maxDatagramSize); + assertRowsEqual(expectedRows(rows), decodeRows(result.packets)); + }); + } + + @Test + public void testBoundedSenderOmittedNonNullableColumnsPreservesRowsAndPacketLimit() throws Exception { + assertMemoryLeak(() -> { + String alpha = repeat('a', 256); + String beta = repeat('b', 192); + String omega = repeat('z', 256); + List rows = Arrays.asList( + row("mix", sender -> sender.table("mix") + .longColumn("x", 1) + .stringColumn("msg", alpha) + .atNow(), + "x", 1L, + "msg", alpha), + row("mix", sender -> sender.table("mix") + .stringColumn("msg", beta) + .atNow(), + "x", Long.MIN_VALUE, + "msg", beta), + row("mix", sender -> sender.table("mix") + .longColumn("x", 3) + .stringColumn("msg", omega) + .atNow(), + "x", 3L, + "msg", omega) + ); + + int maxDatagramSize = fullPacketSize(rows) - 1; + RunResult result = runScenario(rows, maxDatagramSize); + + Assert.assertTrue("expected at least one auto-flush", result.packets.size() > 1); + assertPacketsWithinLimit(result, maxDatagramSize); + assertRowsEqual(expectedRows(rows), decodeRows(result.packets)); + }); + } + + @Test + public void testBoundedSenderOutOfOrderExistingColumnsPreservesRowsAndPacketLimit() throws Exception { + assertMemoryLeak(() -> { + List rows = Arrays.asList( + row("order", sender -> sender.table("order") + .longColumn("a", 1) + .stringColumn("b", "x") + .symbol("c", "alpha") + .atNow(), + "a", 1L, + "b", "x", + "c", "alpha"), + row("order", sender -> sender.table("order") + .symbol("c", "beta") + .stringColumn("b", "y") + .longColumn("a", 2) + .atNow(), + "a", 2L, + "b", "y", + "c", "beta"), + row("order", sender -> sender.table("order") + .stringColumn("b", "z") + .longColumn("a", 3) + .symbol("c", null) + .atNow(), + "a", 3L, + "b", "z", + "c", null) + ); + + int maxDatagramSize = fullPacketSize(rows) - 1; + RunResult result = runScenario(rows, maxDatagramSize); + + Assert.assertTrue("expected at least one auto-flush", result.packets.size() > 1); + assertPacketsWithinLimit(result, maxDatagramSize); + assertRowsEqual(expectedRows(rows), decodeRows(result.packets)); + }); + } + + @Test + public void testBoundedSenderRepeatedOverflowBoundariesWithDistinctSymbolsPreserveRowsAndPacketLimit() throws Exception { + assertMemoryLeak(() -> { + String payload = repeat('p', 256); + List rows = Arrays.asList( + row("t", sender -> sender.table("t") + .symbol("sym", "v0") + .longColumn("x", 0) + .stringColumn("s", payload) + .atNow(), + "sym", "v0", + "x", 0L, + "s", payload), + row("t", sender -> sender.table("t") + .symbol("sym", "v1") + .longColumn("x", 1) + .stringColumn("s", payload) + .atNow(), + "sym", "v1", + "x", 1L, + "s", payload), + row("t", sender -> sender.table("t") + .symbol("sym", "v2") + .longColumn("x", 2) + .stringColumn("s", payload) + .atNow(), + "sym", "v2", + "x", 2L, + "s", payload), + row("t", sender -> sender.table("t") + .symbol("sym", "v3") + .longColumn("x", 3) + .stringColumn("s", payload) + .atNow(), + "sym", "v3", + "x", 3L, + "s", payload), + row("t", sender -> sender.table("t") + .symbol("sym", "v4") + .longColumn("x", 4) + .stringColumn("s", payload) + .atNow(), + "sym", "v4", + "x", 4L, + "s", payload) + ); + int maxDatagramSize = fullPacketSize(rows.subList(0, 2)) - 1; + + RunResult result = runScenario(rows, maxDatagramSize); + + Assert.assertEquals(rows.size(), result.sendCount); + assertPacketsWithinLimit(result, maxDatagramSize); + assertRowsEqual(expectedRows(rows), decodeRows(result.packets)); + }); + } + + @Test + public void testBoundedSenderSchemaFlushThenOmittedNullableColumnsPreservesRows() throws Exception { + assertMemoryLeak(() -> { + List rows = Arrays.asList( + row("schema", sender -> sender.table("schema") + .longColumn("x", 1) + .stringColumn("s", "alpha") + .atNow(), + "x", 1L, + "s", "alpha"), + row("schema", sender -> sender.table("schema") + .symbol("sym", "new") + .longColumn("x", 2) + .stringColumn("s", "beta") + .atNow(), + "sym", "new", + "x", 2L, + "s", "beta"), + row("schema", sender -> sender.table("schema") + .longColumn("x", 3) + .atNow(), + "sym", null, + "x", 3L, + "s", null) + ); + + RunResult result = runScenario(rows, 1024 * 1024); + + Assert.assertEquals(2, result.sendCount); + assertPacketsWithinLimit(result, 1024 * 1024); + assertRowsEqual(expectedRows(rows), decodeRows(result.packets)); + }); + } + + @Test + public void testCancelRowAfterMidRowSchemaChangeDoesNotLeakSchema() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) { + sender.table("t") + .longColumn("a", 1) + .atNow(); + + sender.longColumn("a", 2); + sender.longColumn("b", 3); + Assert.assertEquals(1, nf.sendCount); + + sender.cancelRow(); + sender.longColumn("a", 4).atNow(); + sender.flush(); + } + + Assert.assertEquals(2, nf.sendCount); + assertRowsEqual(Arrays.asList( + decodedRow("t", "a", 1L), + decodedRow("t", "a", 4L) + ), decodeRows(nf.packets)); + }); + } + + @Test + public void testCloseDropsInProgressRowButFlushesCommittedRows() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1)) { + sender.table("t").longColumn("x", 1).atNow(); + sender.longColumn("x", 2); + } + + Assert.assertEquals(1, nf.sendCount); + assertRowsEqual(Arrays.asList(decodedRow("t", "x", 1L)), decodeRows(nf.packets)); + }); + } + + @Test + public void testDuplicateColumnAfterSchemaFlushReplayIsRejected() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) { + sender.table("t") + .longColumn("a", 1) + .atNow(); + + sender.longColumn("a", 2); + sender.longColumn("b", 3); + Assert.assertEquals(1, nf.sendCount); + + assertThrowsContains("column 'a' already set for current row", () -> sender.longColumn("a", 4)); + + sender.cancelRow(); + sender.longColumn("a", 5).atNow(); + sender.flush(); + } + + Assert.assertEquals(2, nf.sendCount); + assertRowsEqual(Arrays.asList( + decodedRow("t", "a", 1L), + decodedRow("t", "a", 5L) + ), decodeRows(nf.packets)); + }); + } + + @Test + public void testEstimateMatchesActualEncodedSize() throws Exception { + assertMemoryLeak(() -> { + auditEstimateWithStableSchemaAndNullableValues(); + auditEstimateAcrossSymbolDictionaryVarintBoundary(); + }); + } + + @Test + public void testFirstRowAllowsMultipleNewColumnsAndEncodesRow() throws Exception { + assertMemoryLeak(() -> { + List rows = Arrays.asList( + row("t", sender -> sender.table("t") + .longColumn("a", 1) + .doubleColumn("b", 2.0) + .stringColumn("c", "x") + .atNow(), + "a", 1L, + "b", 2.0, + "c", "x") + ); + + RunResult result = runScenario(rows, 1024 * 1024); + Assert.assertEquals(1, result.sendCount); + assertRowsEqual(expectedRows(rows), decodeRows(result.packets)); + }); + } + + @Test + public void testFlushPreservesPendingFillStateForCurrentTable() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1)) { + sender.table("t") + .longColumn("a", 1) + .longColumn("b", 2) + .atNow(); + + sender.flush(); + + sender.longColumn("a", 3).atNow(); + sender.flush(); + } + + Assert.assertEquals(2, nf.sendCount); + assertRowsEqual(Arrays.asList( + decodedRow("t", "a", 1L, "b", 2L), + decodedRow("t", "a", 3L, "b", Long.MIN_VALUE) + ), decodeRows(nf.packets)); + }); + } + + @Test + public void testFlushWhileRowInProgressThrowsAndPreservesRow() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1)) { + sender.table("t").longColumn("x", 1); + + assertThrowsContains("Cannot flush buffer while row is in progress", sender::flush); + + sender.atNow(); + sender.flush(); + } + + Assert.assertEquals(1, nf.sendCount); + assertRowsEqual(Arrays.asList(decodedRow("t", "x", 1L)), decodeRows(nf.packets)); + }); + } + + @Test + public void testIrregularArrayRejectedDuringStagingAndSenderRemainsUsable() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) { + sender.table("arrays"); + assertThrowsContains("irregular array shape", () -> + sender.doubleArray("da", new double[][]{{1.0}, {2.0, 3.0}}) + ); + + sender.table("ok"); + sender.longColumn("x", 1).atNow(); + sender.flush(); + } + + Assert.assertEquals(1, nf.sendCount); + assertRowsEqual(Arrays.asList(decodedRow("ok", "x", 1L)), decodeRows(nf.packets)); + }); + } + + @Test + public void testIrregularArrayRejectedDuringStagingDoesNotLeakColumnIntoSameTable() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) { + sender.table("arrays"); + assertThrowsContains("irregular array shape", () -> + sender.doubleArray("bad", new double[][]{{1.0}, {2.0, 3.0}}) + ); + + sender.longColumn("x", 1).atNow(); + sender.flush(); + } + + Assert.assertEquals(1, nf.sendCount); + assertRowsEqual(Arrays.asList(decodedRow("arrays", "x", 1L)), decodeRows(nf.packets)); + }); + } + + @Test + public void testLongArrayWrapperStagingSnapshotsMutation() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024); + LongArray longArray = new LongArray(2, 2)) { + sender.table("arrays"); + + longArray.append(1).append(2).append(3).append(4); + sender.longArray("la", longArray); + + longArray.clear(); + longArray.append(10).append(20).append(30).append(40); + sender.atNow(); + sender.flush(); + } + + assertRowsEqual( + Arrays.asList(decodedRow( + "arrays", + "la", longArrayValue(shape(2, 2), 1, 2, 3, 4) + )), + decodeRows(nf.packets) + ); + }); + } + + @Test + public void testMixingAtNowAndAtMicrosAfterCommittedRowsSplitsDatagramAndPreservesRows() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) { + sender.table("t") + .longColumn("x", 1) + .atNow(); + + sender.longColumn("x", 2); + sender.at(2, ChronoUnit.MICROS); + Assert.assertEquals(1, nf.sendCount); + + sender.flush(); + } + + Assert.assertEquals(2, nf.sendCount); + assertRowsEqual(Arrays.asList( + decodedRow("t", "x", 1L), + decodedRow("t", "x", 2L, "", 2L) + ), decodeRows(nf.packets)); + }); + } + + @Test + public void testNullableArrayReplayKeepsNullArrayStateWithoutReflection() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) { + sender.table("t") + .longArray("la", new long[]{1, 2}) + .doubleArray("da", new double[]{1.0, 2.0}) + .atNow(); + + sender.stageNullLongArrayForTest("la"); + sender.stageNullDoubleArrayForTest("da"); + sender.longColumn("b", 3); + + Assert.assertEquals(1, nf.sendCount); + + sender.atNow(); + + QwpTableBuffer tableBuffer = sender.currentTableBufferForTest(); + Assert.assertNotNull(tableBuffer); + Assert.assertEquals(1, tableBuffer.getRowCount()); + + assertNullableArrayNullState(tableBuffer.getExistingColumn("la", TYPE_LONG_ARRAY)); + assertNullableArrayNullState(tableBuffer.getExistingColumn("da", TYPE_DOUBLE_ARRAY)); + + QwpTableBuffer.ColumnBuffer longColumn = tableBuffer.getExistingColumn("b", TYPE_LONG); + Assert.assertNotNull(longColumn); + Assert.assertEquals(1, longColumn.getSize()); + Assert.assertEquals(1, longColumn.getValueCount()); + Assert.assertEquals(3L, Unsafe.getUnsafe().getLong(longColumn.getDataAddress())); + } + }); + } + + @Test + public void testNullableStringPrefixFlushKeepsNullStateWithoutReflection() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) { + sender.table("t") + .longColumn("x", 1) + .stringColumn("s", "alpha") + .atNow(); + + sender.longColumn("x", 2); + sender.stringColumn("s", null); + sender.longColumn("b", 3); + + Assert.assertEquals(1, nf.sendCount); + + QwpTableBuffer tableBuffer = sender.currentTableBufferForTest(); + Assert.assertNotNull(tableBuffer); + Assert.assertEquals(0, tableBuffer.getRowCount()); + + QwpTableBuffer.ColumnBuffer stringColumn = tableBuffer.getExistingColumn("s", TYPE_STRING); + assertNullableStringNullState(stringColumn); + + QwpTableBuffer.ColumnBuffer longColumn = tableBuffer.getExistingColumn("x", TYPE_LONG); + Assert.assertNotNull(longColumn); + Assert.assertEquals(1, longColumn.getSize()); + Assert.assertEquals(1, longColumn.getValueCount()); + Assert.assertEquals(2L, Unsafe.getUnsafe().getLong(longColumn.getDataAddress())); + + QwpTableBuffer.ColumnBuffer newColumn = tableBuffer.getExistingColumn("b", TYPE_LONG); + Assert.assertNotNull(newColumn); + Assert.assertEquals(1, newColumn.getSize()); + Assert.assertEquals(1, newColumn.getValueCount()); + Assert.assertEquals(3L, Unsafe.getUnsafe().getLong(newColumn.getDataAddress())); + + sender.atNow(); + sender.flush(); + } + + Assert.assertEquals(2, nf.sendCount); + assertRowsEqual(Arrays.asList( + decodedRow("t", "x", 1L, "s", "alpha"), + decodedRow("t", "x", 2L, "s", null, "b", 3L) + ), decodeRows(nf.packets)); + }); + } + + @Test + public void testOversizedArrayRowRejectedUsesActualEncodedSize() throws Exception { + assertMemoryLeak(() -> { + long[] longValues = new long[1024]; + double[] doubleValues = new double[1024]; + for (int i = 0; i < 1024; i++) { + longValues[i] = i; + doubleValues[i] = i + 0.25; + } + + List rows = Arrays.asList( + row("arrays", sender -> sender.table("arrays") + .longColumn("x", 1) + .longArray("la", longValues) + .doubleArray("da", doubleValues) + .atNow(), + "x", 1L, + "la", longArrayValue(shape(1024), longValues), + "da", doubleArrayValue(shape(1024), doubleValues)) + ); + int maxDatagramSize = fullPacketSize(rows) - 1; + + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) { + sender.table("arrays"); + assertThrowsContains("single row exceeds maximum datagram size", () -> + sender.longColumn("x", 1) + .longArray("la", longValues) + .doubleArray("da", doubleValues) + .atNow() + ); + } + + Assert.assertEquals(0, nf.sendCount); + }); + } + + @Test + public void testOversizedRowAfterMidRowSchemaChangeCancelDoesNotLeakSchema() throws Exception { + assertMemoryLeak(() -> { + String large = repeat('x', 5000); + List oversizedRow = Arrays.asList( + row("t", sender -> sender.table("t") + .longColumn("a", 2) + .stringColumn("s", large) + .atNow(), + "a", 2L, + "s", large) + ); + int maxDatagramSize = fullPacketSize(oversizedRow) - 1; + + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) { + sender.table("t") + .longColumn("a", 1) + .atNow(); + + sender.longColumn("a", 2); + assertThrowsContains("single row exceeds maximum datagram size", () -> { + sender.stringColumn("s", large); + sender.atNow(); + }); + Assert.assertEquals(1, nf.sendCount); + + sender.cancelRow(); + sender.longColumn("a", 3).atNow(); + sender.flush(); + } + + Assert.assertEquals(2, nf.sendCount); + assertRowsEqual(Arrays.asList( + decodedRow("t", "a", 1L), + decodedRow("t", "a", 3L) + ), decodeRows(nf.packets)); + }); + } + + @Test + public void testOversizedSingleRowRejectedAfterReplayUsesActualEncodedSize() throws Exception { + assertMemoryLeak(() -> { + String small = repeat('s', 32); + String large = repeat('x', 5000); + List largeRow = Arrays.asList( + row("t", sender -> sender.table("t") + .longColumn("x", 2) + .stringColumn("s", large) + .atNow(), + "x", 2L, + "s", large) + ); + int maxDatagramSize = fullPacketSize(largeRow) - 1; + + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) { + sender.table("t") + .longColumn("x", 1) + .stringColumn("s", small) + .atNow(); + + assertThrowsContains("single row exceeds maximum datagram size", () -> + sender.longColumn("x", 2) + .stringColumn("s", large) + .atNow() + ); + } + + Assert.assertEquals(1, nf.sendCount); + assertPacketsWithinLimit(new RunResult(nf.packets, nf.lengths, nf.sendCount), maxDatagramSize); + assertRowsEqual( + Arrays.asList(decodedRow("t", "x", 1L, "s", small)), + decodeRows(nf.packets) + ); + }); + } + + @Test + public void testOversizedSingleRowRejectedBeforeReplayUsesActualEncodedSize() throws Exception { + assertMemoryLeak(() -> { + String large = repeat('x', 5000); + List rows = Arrays.asList( + row("t", sender -> sender.table("t") + .longColumn("x", 1) + .stringColumn("s", large) + .atNow(), + "x", 1L, + "s", large) + ); + int maxDatagramSize = fullPacketSize(rows) - 1; + + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) { + sender.table("t"); + assertThrowsContains("single row exceeds maximum datagram size", () -> + sender.longColumn("x", 1) + .stringColumn("s", large) + .atNow() + ); + } + + Assert.assertEquals(0, nf.sendCount); + }); + } + + @Test + public void testRepeatedUtf8SymbolsAcrossRowsPreserveRows() throws Exception { + assertMemoryLeak(() -> { + List rows = Arrays.asList( + row("sym", sender -> sender.table("sym") + .longColumn("x", 1) + .symbol("sym", "東京") + .atNow(), + "x", 1L, + "sym", "東京"), + row("sym", sender -> sender.table("sym") + .longColumn("x", 2) + .symbol("sym", "東京") + .atNow(), + "x", 2L, + "sym", "東京"), + row("sym", sender -> sender.table("sym") + .longColumn("x", 3) + .symbol("sym", "Αθηνα") + .atNow(), + "x", 3L, + "sym", "Αθηνα") + ); + + RunResult result = runScenario(rows, 1024 * 1024); + Assert.assertEquals(1, result.sendCount); + assertRowsEqual(expectedRows(rows), decodeRows(result.packets)); + }); + } + + @Test + public void testSchemaChangeAfterOutOfOrderExistingColumnsPreservesRows() throws Exception { + assertMemoryLeak(() -> { + List rows = Arrays.asList( + row("schema", sender -> sender.table("schema") + .longColumn("a", 1) + .stringColumn("b", "x") + .atNow(), + "a", 1L, + "b", "x"), + row("schema", sender -> sender.table("schema") + .symbol("c", "new") + .stringColumn("b", "y") + .longColumn("a", 2) + .atNow(), + "a", 2L, + "b", "y", + "c", "new"), + row("schema", sender -> sender.table("schema") + .symbol("c", "next") + .longColumn("a", 3) + .stringColumn("b", "z") + .atNow(), + "a", 3L, + "b", "z", + "c", "next") + ); + + RunResult result = runScenario(rows, 1024 * 1024); + + Assert.assertEquals(2, result.sendCount); + assertRowsEqual(expectedRows(rows), decodeRows(result.packets)); + }); + } + + @Test + public void testSchemaChangeMidRowAllowsMultipleNewColumnsAfterReplay() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) { + sender.table("t") + .longColumn("a", 1) + .atNow(); + + sender.longColumn("a", 2); + sender.longColumn("b", 3); + Assert.assertEquals(1, nf.sendCount); + + sender.stringColumn("c", "x"); + sender.longColumn("d", 4); + Assert.assertEquals(1, nf.sendCount); + + sender.atNow(); + sender.flush(); + } + + Assert.assertEquals(2, nf.sendCount); + assertRowsEqual(Arrays.asList( + decodedRow("t", "a", 1L), + decodedRow("t", "a", 2L, "b", 3L, "c", "x", "d", 4L) + ), decodeRows(nf.packets)); + }); + } + + @Test + public void testSchemaChangeMidRowAllowsMultipleNewColumnsAfterReplayWithoutDatagramTracking() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1)) { + sender.table("t") + .longColumn("a", 1) + .atNow(); + + sender.longColumn("a", 2); + sender.longColumn("b", 3); + Assert.assertEquals(1, nf.sendCount); + + sender.stringColumn("c", "x"); + sender.longColumn("d", 4); + Assert.assertEquals(1, nf.sendCount); + + sender.atNow(); + sender.flush(); + } + + Assert.assertEquals(2, nf.sendCount); + assertRowsEqual(Arrays.asList( + decodedRow("t", "a", 1L), + decodedRow("t", "a", 2L, "b", 3L, "c", "x", "d", 4L) + ), decodeRows(nf.packets)); + }); + } + + @Test + public void testSchemaChangeMidRowFlushesImmediatelyAndPreservesRows() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) { + sender.table("t") + .longColumn("a", 1) + .atNow(); + + sender.longColumn("a", 2); + sender.longColumn("b", 3); + Assert.assertEquals(1, nf.sendCount); + + sender.atNow(); + Assert.assertEquals(1, nf.sendCount); + sender.flush(); + } + + Assert.assertEquals(2, nf.sendCount); + assertRowsEqual(Arrays.asList( + decodedRow("t", "a", 1L), + decodedRow("t", "a", 2L, "b", 3L) + ), decodeRows(nf.packets)); + }); + } + + @Test + public void testSchemaChangeMidRowPreservesExistingArrayValues() throws Exception { + assertMemoryLeak(() -> { + double[] first = {1.0, 2.0}; + double[] second = {3.5, 4.5, 5.5}; + + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) { + sender.table("t") + .longColumn("a", 1) + .doubleArray("da", first) + .atNow(); + + sender.longColumn("a", 2); + sender.doubleArray("da", second); + sender.longColumn("b", 3); + Assert.assertEquals(1, nf.sendCount); + + sender.atNow(); + sender.flush(); + } + + Assert.assertEquals(2, nf.sendCount); + assertRowsEqual(Arrays.asList( + decodedRow("t", "a", 1L, "da", doubleArrayValue(shape(2), first)), + decodedRow("t", "a", 2L, "da", doubleArrayValue(shape(3), second), "b", 3L) + ), decodeRows(nf.packets)); + }); + } + + @Test + public void testSchemaChangeMidRowPreservesExistingStringAndSymbolValues() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) { + sender.table("t") + .longColumn("a", 1) + .stringColumn("s", "alpha") + .symbol("sym", "one") + .atNow(); + + sender.longColumn("a", 2); + sender.stringColumn("s", "beta"); + sender.symbol("sym", "two"); + sender.longColumn("b", 3); + Assert.assertEquals(1, nf.sendCount); + + sender.atNow(); + sender.flush(); + } + + Assert.assertEquals(2, nf.sendCount); + assertRowsEqual(Arrays.asList( + decodedRow("t", "a", 1L, "s", "alpha", "sym", "one"), + decodedRow("t", "a", 2L, "s", "beta", "sym", "two", "b", 3L) + ), decodeRows(nf.packets)); + }); + } + + @Test + public void testSchemaChangeWithCommittedRowsFlushesImmediately() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) { + sender.table("t") + .longColumn("a", 1) + .atNow(); + Assert.assertEquals(0, nf.sendCount); + + sender.longColumn("b", 2); + Assert.assertEquals(1, nf.sendCount); + + sender.atNow(); + Assert.assertEquals(1, nf.sendCount); + + sender.flush(); + Assert.assertEquals(2, nf.sendCount); + } + }); + } + + @Test + public void testSimpleLongRowUsesScatterSendPath() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1)) { + sender.table("t").longColumn("x", 42L).atNow(); + sender.flush(); + } + + Assert.assertEquals(1, nf.sendCount); + Assert.assertEquals(1, nf.scatterSendCount); + Assert.assertEquals(0, nf.rawSendCount); + Assert.assertTrue("expected multiple segments for header/schema/data", nf.segmentCounts.get(0) > 1); + assertRowsEqual(Arrays.asList(decodedRow("t", "x", 42L)), decodeRows(nf.packets)); + }); + } + + @Test + public void testSwitchTableWhileRowInProgressThrowsAndPreservesRows() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1)) { + sender.table("t1").longColumn("x", 1); + + assertThrowsContains("Cannot switch tables while row is in progress", + () -> sender.table("t2")); + + sender.atNow(); + sender.table("t2").longColumn("y", 2).atNow(); + sender.flush(); + } + + Assert.assertEquals(2, nf.sendCount); + assertRowsEqualIgnoringOrder( + Arrays.asList(decodedRow("t1", "x", 1L), decodedRow("t2", "y", 2L)), + decodeRows(nf.packets) + ); + }); + } + + @Test + public void testSymbolBoundary127To128PreservesRowsAndPacketLimit() throws Exception { + assertMemoryLeak(() -> { + List rows = new ArrayList<>(130); + for (int i = 0; i < 130; i++) { + final int value = i; + rows.add(row("t", sender -> sender.table("t") + .symbol("sym", "v" + value) + .longColumn("x", value) + .atNow(), + "sym", "v" + value, + "x", (long) value)); + } + + int maxDatagramSize = fullPacketSize(rows) - 1; + RunResult result = runScenario(rows, maxDatagramSize); + + Assert.assertTrue("expected at least one auto-flush", result.packets.size() > 1); + assertPacketsWithinLimit(result, maxDatagramSize); + assertRowsEqual(expectedRows(rows), decodeRows(result.packets)); + }); + } + + @Test + public void testSymbolBoundary16383To16384PreservesRowsAndPacketLimit() throws Exception { + assertMemoryLeak(() -> { + List rows = new ArrayList<>(16_390); + for (int i = 0; i < 16_390; i++) { + final int value = i; + rows.add(row("t", sender -> sender.table("t") + .symbol("sym", "v" + value) + .longColumn("x", value) + .atNow(), + "sym", "v" + value, + "x", (long) value)); + } + + int maxDatagramSize = fullPacketSize(rows) - 1; + RunResult result = runScenario(rows, maxDatagramSize); + + Assert.assertTrue("expected at least one auto-flush", result.packets.size() > 1); + assertPacketsWithinLimit(result, maxDatagramSize); + assertRowsEqual(expectedRows(rows), decodeRows(result.packets)); + }); + } + + @Test + public void testSymbolPrefixFlushKeepsSingleRetainedDictionaryEntry() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) { + sender.table("t") + .symbol("sym", "alpha") + .longColumn("x", 1) + .atNow(); + + sender.symbol("sym", "beta"); + sender.longColumn("x", 2); + sender.longColumn("b", 3); + + Assert.assertEquals(1, nf.sendCount); + + QwpTableBuffer tableBuffer = sender.currentTableBufferForTest(); + Assert.assertNotNull(tableBuffer); + Assert.assertEquals(0, tableBuffer.getRowCount()); + + QwpTableBuffer.ColumnBuffer symbolColumn = tableBuffer.getExistingColumn("sym", TYPE_SYMBOL); + Assert.assertNotNull(symbolColumn); + Assert.assertEquals(1, symbolColumn.getSize()); + Assert.assertEquals(1, symbolColumn.getValueCount()); + Assert.assertEquals(1, symbolColumn.getSymbolDictionarySize()); + Assert.assertEquals("beta", symbolColumn.getSymbolValue(0)); + Assert.assertTrue(symbolColumn.hasSymbol("beta")); + Assert.assertFalse(symbolColumn.hasSymbol("alpha")); + Assert.assertEquals(0, Unsafe.getUnsafe().getInt(symbolColumn.getDataAddress())); + + sender.atNow(); + sender.flush(); + } + + Assert.assertEquals(2, nf.sendCount); + assertRowsEqual(Arrays.asList( + decodedRow("t", "sym", "alpha", "x", 1L), + decodedRow("t", "sym", "beta", "x", 2L, "b", 3L) + ), decodeRows(nf.packets)); + }); + } + + @Test + public void testTimestampOnlyRows() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1)) { + // at() with no other columns: designated timestamp is staged + sender.table("t").at(1_000L, ChronoUnit.MICROS); + // atNow() with no other columns: server assigns the timestamp + sender.table("t").atNow(); + sender.flush(); + } + + List rows = decodeRows(nf.packets); + Assert.assertEquals("expected 2 timestamp-only rows", 2, rows.size()); + assertRowsEqual(Arrays.asList( + decodedRow("t", "", 1_000L), + decodedRow("t", "", null) + ), rows); + }); + } + + @Test + public void testUnboundedSenderOmittedNullableAndNonNullableColumnsPreservesRows() throws Exception { + assertMemoryLeak(() -> { + List rows = Arrays.asList( + row("t", sender -> sender.table("t") + .longColumn("x", 1) + .stringColumn("s", "alpha") + .symbol("sym", "one") + .atNow(), + "x", 1L, + "s", "alpha", + "sym", "one"), + row("t", sender -> sender.table("t") + .stringColumn("s", "beta") + .atNow(), + "x", Long.MIN_VALUE, + "s", "beta", + "sym", null), + row("t", sender -> sender.table("t") + .longColumn("x", 3) + .atNow(), + "x", 3L, + "s", null, + "sym", null) + ); + + RunResult result = runScenario(rows, 0); + + Assert.assertEquals(1, result.sendCount); + assertRowsEqual(expectedRows(rows), decodeRows(result.packets)); + }); + } + + @Test + public void testUnboundedSenderWideSchemaWithLowIndexWritePreservesRows() throws Exception { + assertMemoryLeak(() -> { + List rows = Arrays.asList( + row("wide", sender -> sender.table("wide") + .longColumn("c0", 0) + .longColumn("c1", 1) + .longColumn("c2", 2) + .longColumn("c3", 3) + .longColumn("c4", 4) + .longColumn("c5", 5) + .longColumn("c6", 6) + .longColumn("c7", 7) + .longColumn("c8", 8) + .longColumn("c9", 9) + .atNow(), + "c0", 0L, + "c1", 1L, + "c2", 2L, + "c3", 3L, + "c4", 4L, + "c5", 5L, + "c6", 6L, + "c7", 7L, + "c8", 8L, + "c9", 9L), + row("wide", sender -> sender.table("wide") + .longColumn("c0", 10) + .atNow(), + "c0", 10L, + "c1", Long.MIN_VALUE, + "c2", Long.MIN_VALUE, + "c3", Long.MIN_VALUE, + "c4", Long.MIN_VALUE, + "c5", Long.MIN_VALUE, + "c6", Long.MIN_VALUE, + "c7", Long.MIN_VALUE, + "c8", Long.MIN_VALUE, + "c9", Long.MIN_VALUE) + ); + + RunResult result = runScenario(rows, 0); + + Assert.assertEquals(1, result.sendCount); + assertRowsEqual(expectedRows(rows), decodeRows(result.packets)); + }); + } + + @Test + public void testUtf8StringAndSymbolStagingSupportsCancelAndPacketSizing() throws Exception { + assertMemoryLeak(() -> { + String msg1 = "Gruesse 東京"; + String msg2 = "Privet 👋 kosme"; + List rows = Arrays.asList( + row("utf8", sender -> sender.table("utf8") + .longColumn("x", 1) + .symbol("sym", "東京") + .stringColumn("msg", msg1) + .atNow(), + "x", 1L, + "sym", "東京", + "msg", msg1), + row("utf8", sender -> sender.table("utf8") + .longColumn("x", 2) + .symbol("sym", "Αθηνα") + .stringColumn("msg", msg2) + .atNow(), + "x", 2L, + "sym", "Αθηνα", + "msg", msg2) + ); + int maxDatagramSize = fullPacketSize(rows) - 1; + + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) { + sender.table("utf8") + .longColumn("x", 0) + .symbol("sym", "キャンセル") + .stringColumn("msg", "should not ship 👎"); + sender.cancelRow(); + + sender.longColumn("x", 1) + .symbol("sym", "東京") + .stringColumn("msg", msg1) + .atNow(); + sender.longColumn("x", 2) + .symbol("sym", "Αθηνα") + .stringColumn("msg", msg2) + .atNow(); + sender.flush(); + } + + RunResult result = new RunResult(nf.packets, nf.lengths, nf.sendCount); + Assert.assertTrue("expected at least one auto-flush", result.packets.size() > 1); + assertPacketsWithinLimit(result, maxDatagramSize); + assertRowsEqual(expectedRows(rows), decodeRows(nf.packets)); + }); + } + + private static void assertEstimateAtLeastActual(List rows) throws Exception { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) { + for (int i = 0; i < rows.size(); i++) { + rows.get(i).writer.accept(sender); + long estimate = sender.committedDatagramEstimateForTest(); + long actual = fullPacketSize(rows.subList(0, i + 1)); + Assert.assertTrue( + "row " + i + " estimate underflow: estimate=" + estimate + ", actual=" + actual, + estimate >= actual + ); + } + } + } + + private static void assertNullableArrayNullState(QwpTableBuffer.ColumnBuffer column) { + Assert.assertNotNull(column); + Assert.assertEquals(1, column.getSize()); + Assert.assertEquals(0, column.getValueCount()); + Assert.assertTrue(column.usesNullBitmap()); + Assert.assertTrue(column.isNull(0)); + Assert.assertEquals(0, column.getArrayShapeOffset()); + Assert.assertEquals(0, column.getArrayDataOffset()); + } + + private static void assertNullableStringNullState(QwpTableBuffer.ColumnBuffer column) { + Assert.assertNotNull(column); + Assert.assertEquals(1, column.getSize()); + Assert.assertEquals(0, column.getValueCount()); + Assert.assertTrue(column.usesNullBitmap()); + Assert.assertTrue(column.isNull(0)); + Assert.assertEquals(0, column.getStringDataSize()); + long offsetsAddress = column.getStringOffsetsAddress(); + Assert.assertTrue(offsetsAddress != 0); + Assert.assertEquals(0, Unsafe.getUnsafe().getInt(offsetsAddress)); + } + + private static void assertPacketsWithinLimit(RunResult result, int maxDatagramSize) { + for (int i = 0; i < result.lengths.size(); i++) { + int len = result.lengths.get(i); + Assert.assertTrue( + "packet " + i + " exceeds maxDatagramSize: " + len + " > " + maxDatagramSize, + len <= maxDatagramSize + ); + } + } + + private static void assertRowsEqual(List expected, List actual) { + Assert.assertEquals("row count mismatch", expected.size(), actual.size()); + for (int i = 0; i < expected.size(); i++) { + Assert.assertEquals("row mismatch at index " + i, expected.get(i), actual.get(i)); + } + } + + private static void assertRowsEqualIgnoringOrder(List expected, List actual) { + Map counts = new HashMap<>(); + for (DecodedRow row : expected) { + counts.merge(row, 1, Integer::sum); + } + for (DecodedRow row : actual) { + Integer count = counts.get(row); + if (count == null) { + Assert.fail("unexpected row: " + row); + } + if (count == 1) { + counts.remove(row); + } else { + counts.put(row, count - 1); + } + } + if (!counts.isEmpty()) { + Assert.fail("missing rows: " + counts); + } + } + + private static void assertThrowsContains(String expected, ThrowingRunnable runnable) { + try { + runnable.run(); + Assert.fail("Expected exception containing: " + expected); + } catch (LineSenderException e) { + Assert.assertTrue("Expected message to contain '" + expected + "' but got: " + e.getMessage(), + e.getMessage().contains(expected)); + } catch (Exception e) { + throw new AssertionError("Unexpected exception type", e); + } + } + + private static void auditEstimateAcrossSymbolDictionaryVarintBoundary() throws Exception { + ArrayList rows = new ArrayList<>(); + for (int i = 0; i < 160; i++) { + final int rowId = i; + rows.add(row( + "sym_audit", + sender -> sender.table("sym_audit") + .longColumn("x", rowId) + .symbol("sym", "sym-" + rowId) + .atNow(), + "x", (long) rowId, + "sym", "sym-" + rowId + )); + } + assertEstimateAtLeastActual(rows); + } + + private static void auditEstimateWithStableSchemaAndNullableValues() throws Exception { + ArrayList rows = new ArrayList<>(); + for (int i = 0; i < 96; i++) { + final int rowId = i; + final String stringValue = (i & 1) == 0 ? "tokyo-" + i + "-" + repeat('x', (i % 31) + 1) : null; + final long[] longArray = i % 3 == 0 ? new long[]{i, i + 1L, i + 2L} : null; + final double[][] doubleArray = i % 5 == 0 ? new double[][]{{i + 0.5, i + 1.5}, {i + 2.5, i + 3.5}} : null; + final Decimal64 decimal64 = i % 7 == 0 ? Decimal64.fromLong(i * 100L + 7, 2) : null; + final Decimal128 decimal128 = i % 11 == 0 ? Decimal128.fromLong(i * 1000L + 11, 4) : null; + final Decimal256 decimal256 = i % 13 == 0 ? Decimal256.fromLong(i * 10000L + 13, 3) : null; + + rows.add(row( + "audit", + sender -> { + sender.table("audit") + .longColumn("l", rowId) + .doubleColumn("d", rowId + 0.25) + .symbol("sym", "stable"); + if (stringValue != null) { + sender.stringColumn("s", stringValue); + } + if (longArray != null) { + sender.longArray("la", longArray); + } + if (doubleArray != null) { + sender.doubleArray("da", doubleArray); + } + if (decimal64 != null) { + sender.decimalColumn("d64", decimal64); + } + if (decimal128 != null) { + sender.decimalColumn("d128", decimal128); + } + if (decimal256 != null) { + sender.decimalColumn("d256", decimal256); + } + sender.at(rowId + 1L, ChronoUnit.MICROS); + }, + "l", (long) rowId, + "d", rowId + 0.25, + "sym", "stable", + "s", stringValue, + "la", longArray == null ? null : longArrayValue(shape(longArray.length), longArray), + "da", doubleArray == null ? null : doubleArrayValue(shape(doubleArray.length, doubleArray[0].length), flatten(doubleArray)), + "d64", decimal64 == null ? null : decimal(i * 100L + 7, 2), + "d128", decimal128 == null ? null : decimal(i * 1000L + 11, 4), + "d256", decimal256 == null ? null : decimal(i * 10000L + 13, 3), + "", rowId + 1L + )); + } + assertEstimateAtLeastActual(rows); + } + + private static LinkedHashMap columns(Object... kvs) { + if ((kvs.length & 1) != 0) { + throw new IllegalArgumentException("key/value pairs expected"); + } + LinkedHashMap columns = new LinkedHashMap<>(); + for (int i = 0; i < kvs.length; i += 2) { + columns.put((String) kvs[i], kvs[i + 1]); + } + return columns; + } + + private static BigDecimal decimal(long unscaled, int scale) { + return BigDecimal.valueOf(unscaled, scale); + } + + private static List decodeRows(List packets) { + ArrayList rows = new ArrayList<>(); + for (byte[] packet : packets) { + rows.addAll(new DatagramDecoder(packet).decode()); + } + return rows; + } + + private static DecodedRow decodedRow(String table, Object... kvs) { + return new DecodedRow(table, columns(kvs)); + } + + private static DoubleArrayValue doubleArrayValue(int[] shape, double... values) { + ArrayList dims = new ArrayList<>(shape.length); + for (int dim : shape) { + dims.add(dim); + } + ArrayList elems = new ArrayList<>(values.length); + for (double value : values) { + elems.add(value); + } + return new DoubleArrayValue(dims, elems); + } + + private static List expectedRows(List rows) { + ArrayList expected = new ArrayList<>(rows.size()); + for (ScenarioRow row : rows) { + expected.add(row.expected); + } + return expected; + } + + private static double[] flatten(double[][] matrix) { + int size = 0; + for (double[] row : matrix) { + size += row.length; + } + double[] flat = new double[size]; + int offset = 0; + for (double[] row : matrix) { + System.arraycopy(row, 0, flat, offset, row.length); + offset += row.length; + } + return flat; + } + + private static long[] flatten(long[][] matrix) { + int size = 0; + for (long[] row : matrix) { + size += row.length; + } + long[] flat = new long[size]; + int offset = 0; + for (long[] row : matrix) { + System.arraycopy(row, 0, flat, offset, row.length); + offset += row.length; + } + return flat; + } + + private static int fullPacketSize(List rows) throws Exception { + RunResult result = runScenario(rows, 0); + Assert.assertEquals("expected a single unbounded packet", 1, result.packets.size()); + return result.lengths.get(0); + } + + private static LongArrayValue longArrayValue(int[] shape, long... values) { + ArrayList dims = new ArrayList<>(shape.length); + for (int dim : shape) { + dims.add(dim); + } + ArrayList elems = new ArrayList<>(values.length); + for (long value : values) { + elems.add(value); + } + return new LongArrayValue(dims, elems); + } + + private static String repeat(char value, int count) { + char[] chars = new char[count]; + Arrays.fill(chars, value); + return new String(chars); + } + + private static ScenarioRow row(String table, ThrowingConsumer writer, Object... kvs) { + return new ScenarioRow(decodedRow(table, kvs), writer); + } + + private static RunResult runScenario(List rows, int maxDatagramSize) throws Exception { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + if (maxDatagramSize > 0) { + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) { + for (ScenarioRow row : rows) { + row.writer.accept(sender); + } + sender.flush(); + } + } else { + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1)) { + for (ScenarioRow row : rows) { + row.writer.accept(sender); + } + sender.flush(); + } + } + return new RunResult(nf.packets, nf.lengths, nf.sendCount); + } + + private static int[] shape(int... dims) { + return dims; + } + + @FunctionalInterface + private interface ThrowingConsumer { + void accept(T value) throws Exception; + } + + @FunctionalInterface + private interface ThrowingRunnable { + void run() throws Exception; + } + + private static final class CapturingNetworkFacade extends NetworkFacadeImpl { + private static final int SEGMENT_SIZE = 16; + + private final List lengths = new ArrayList<>(); + private final List packets = new ArrayList<>(); + private final List segmentCounts = new ArrayList<>(); + private int rawSendCount; + private int scatterSendCount; + private int sendCount; + + @Override + public int close(int fd) { + return 0; + } + + @Override + public void freeSockAddr(long pSockaddr) { + // no-op + } + + @Override + public int sendToRaw(int fd, long lo, int len, long socketAddress) { + byte[] packet = new byte[len]; + Unsafe.getUnsafe().copyMemory(null, lo, packet, Unsafe.BYTE_OFFSET, len); + packets.add(packet); + lengths.add(len); + rawSendCount++; + sendCount++; + return len; + } + + @Override + public int sendToRawScatter(int fd, long segmentsPtr, int segmentCount, long socketAddress) { + int packetLength = 0; + long segmentPtr = segmentsPtr; + for (int i = 0; i < segmentCount; i++) { + packetLength += (int) Unsafe.getUnsafe().getLong(segmentPtr + 8); + segmentPtr += SEGMENT_SIZE; + } + + byte[] packet = new byte[packetLength]; + int offset = 0; + segmentPtr = segmentsPtr; + for (int i = 0; i < segmentCount; i++) { + long dataAddress = Unsafe.getUnsafe().getLong(segmentPtr); + int dataLength = (int) Unsafe.getUnsafe().getLong(segmentPtr + 8); + Unsafe.getUnsafe().copyMemory(null, dataAddress, packet, Unsafe.BYTE_OFFSET + offset, dataLength); + offset += dataLength; + segmentPtr += SEGMENT_SIZE; + } + + packets.add(packet); + lengths.add(packetLength); + segmentCounts.add(segmentCount); + scatterSendCount++; + sendCount++; + return packetLength; + } + + @Override + public int setMulticastInterface(int fd, int ipv4Address) { + return 0; + } + + @Override + public int setMulticastTtl(int fd, int ttl) { + return 0; + } + + @Override + public long sockaddr(int address, int port) { + return 1; + } + + @Override + public int socketUdp() { + return 1; + } + } + + private static final class ColumnValues { + private final Object[] rows; + + private ColumnValues(Object[] rows) { + this.rows = rows; + } + } + + private static final class DatagramDecoder { + private final PacketReader reader; + + private DatagramDecoder(byte[] packet) { + this.reader = new PacketReader(packet); + } + + private static int countNulls(boolean[] nulls) { + int count = 0; + for (boolean value : nulls) { + if (value) { + count++; + } + } + return count; + } + + private List decode() { + Assert.assertEquals(MAGIC_MESSAGE, reader.readIntLE()); + Assert.assertEquals(VERSION_1, reader.readByte()); + reader.readByte(); + int tableCount = reader.readUnsignedShortLE(); + int payloadLength = reader.readIntLE(); + Assert.assertEquals(reader.length() - HEADER_SIZE, payloadLength); + + ArrayList rows = new ArrayList<>(); + for (int table = 0; table < tableCount; table++) { + rows.addAll(decodeTable()); + } + Assert.assertEquals(reader.length(), reader.position()); + return rows; + } + + private void decodeBooleans(Object[] values, boolean[] nulls, int valueCount) { + byte[] packed = reader.readBytes((valueCount + 7) / 8); + int valueIndex = 0; + for (int row = 0; row < values.length; row++) { + if (nulls[row]) { + values[row] = null; + } else { + int byteIndex = valueIndex >>> 3; + int bitIndex = valueIndex & 7; + values[row] = (packed[byteIndex] & (1 << bitIndex)) != 0; + valueIndex++; + } + } + } + + private ColumnValues decodeColumn(QwpColumnDef def, int rowCount) { + boolean hasNullBitmap = reader.readByte() != 0; + boolean[] nulls = hasNullBitmap ? reader.readNullBitmap(rowCount) : new boolean[rowCount]; + int nullCount = 0; + for (boolean isNull : nulls) { + if (isNull) { + nullCount++; + } + } + int valueCount = rowCount - nullCount; + Object[] values = new Object[rowCount]; + + switch (def.getTypeCode()) { + case TYPE_BOOLEAN: + decodeBooleans(values, nulls, valueCount); + break; + case TYPE_DECIMAL64: + decodeDecimals(values, nulls, valueCount, 8); + break; + case TYPE_DECIMAL128: + decodeDecimals(values, nulls, valueCount, 16); + break; + case TYPE_DECIMAL256: + decodeDecimals(values, nulls, valueCount, 32); + break; + case TYPE_DOUBLE: + decodeDoubles(values, nulls, valueCount); + break; + case TYPE_DOUBLE_ARRAY: + decodeDoubleArrays(values, nulls, valueCount); + break; + case TYPE_LONG: + case TYPE_TIMESTAMP: + case TYPE_TIMESTAMP_NANOS: + decodeLongs(values, nulls, valueCount); + break; + case TYPE_LONG_ARRAY: + decodeLongArrays(values, nulls, valueCount); + break; + case TYPE_STRING: + case TYPE_VARCHAR: + decodeStrings(values, nulls, valueCount); + break; + case TYPE_SYMBOL: + decodeSymbols(values, nulls, valueCount); + break; + default: + throw new AssertionError("Unsupported test decoder type: " + def.getTypeCode()); + } + + return new ColumnValues(values); + } + + private void decodeDecimals(Object[] values, boolean[] nulls, int valueCount, int width) { + int scale = reader.readByte(); + int valueIndex = 0; + for (int row = 0; row < values.length; row++) { + if (nulls[row]) { + values[row] = null; + } else { + byte[] bytes = reader.readBytes(width); + values[row] = new BigDecimal(new BigInteger(bytes), scale); + valueIndex++; + } + } + Assert.assertEquals(valueCount, valueIndex); + } + + private void decodeDoubleArrays(Object[] values, boolean[] nulls, int valueCount) { + int valueIndex = 0; + for (int row = 0; row < values.length; row++) { + if (nulls[row]) { + values[row] = null; + continue; + } + int dims = reader.readUnsignedByte(); + ArrayList shape = new ArrayList<>(dims); + int elementCount = 1; + for (int d = 0; d < dims; d++) { + int dim = reader.readIntLE(); + shape.add(dim); + elementCount = Math.multiplyExact(elementCount, dim); + } + ArrayList elements = new ArrayList<>(elementCount); + for (int i = 0; i < elementCount; i++) { + elements.add(reader.readDoubleLE()); + } + values[row] = new DoubleArrayValue(shape, elements); + valueIndex++; + } + Assert.assertEquals(valueCount, valueIndex); + } + + private void decodeDoubles(Object[] values, boolean[] nulls, int valueCount) { + int valueIndex = 0; + for (int row = 0; row < values.length; row++) { + if (nulls[row]) { + values[row] = null; + } else { + values[row] = reader.readDoubleLE(); + valueIndex++; + } + } + Assert.assertEquals(valueCount, valueIndex); + } + + private void decodeLongArrays(Object[] values, boolean[] nulls, int valueCount) { + int valueIndex = 0; + for (int row = 0; row < values.length; row++) { + if (nulls[row]) { + values[row] = null; + continue; + } + int dims = reader.readUnsignedByte(); + ArrayList shape = new ArrayList<>(dims); + int elementCount = 1; + for (int d = 0; d < dims; d++) { + int dim = reader.readIntLE(); + shape.add(dim); + elementCount = Math.multiplyExact(elementCount, dim); + } + ArrayList elements = new ArrayList<>(elementCount); + for (int i = 0; i < elementCount; i++) { + elements.add(reader.readLongLE()); + } + values[row] = new LongArrayValue(shape, elements); + valueIndex++; + } + Assert.assertEquals(valueCount, valueIndex); + } + + private void decodeLongs(Object[] values, boolean[] nulls, int valueCount) { + int valueIndex = 0; + for (int row = 0; row < values.length; row++) { + if (nulls[row]) { + values[row] = null; + } else { + values[row] = reader.readLongLE(); + valueIndex++; + } + } + Assert.assertEquals(valueCount, valueIndex); + } + + private void decodeStrings(Object[] values, boolean[] nulls, int valueCount) { + int[] offsets = new int[valueCount + 1]; + for (int i = 0; i <= valueCount; i++) { + offsets[i] = reader.readIntLE(); + } + byte[] data = reader.readBytes(offsets[valueCount]); + + int valueIndex = 0; + for (int row = 0; row < values.length; row++) { + if (nulls[row]) { + values[row] = null; + } else { + int start = offsets[valueIndex]; + int end = offsets[valueIndex + 1]; + values[row] = new String(data, start, end - start, StandardCharsets.UTF_8); + valueIndex++; + } + } + Assert.assertEquals(valueCount, valueIndex); + } + + private void decodeSymbols(Object[] values, boolean[] nulls, int valueCount) { + int dictSize = (int) reader.readVarint(); + String[] dict = new String[dictSize]; + for (int i = 0; i < dictSize; i++) { + dict[i] = reader.readString(); + } + + int valueIndex = 0; + for (int row = 0; row < values.length; row++) { + if (nulls[row]) { + values[row] = null; + } else { + int index = (int) reader.readVarint(); + values[row] = dict[index]; + valueIndex++; + } + } + Assert.assertEquals(valueCount, valueIndex); + } + + private List decodeTable() { + String tableName = reader.readString(); + int rowCount = (int) reader.readVarint(); + int columnCount = (int) reader.readVarint(); + Assert.assertEquals(SCHEMA_MODE_FULL, reader.readByte()); + + QwpColumnDef[] defs = new QwpColumnDef[columnCount]; + for (int i = 0; i < columnCount; i++) { + defs[i] = new QwpColumnDef(reader.readString(), reader.readByte()); + } + + ColumnValues[] columns = new ColumnValues[columnCount]; + for (int i = 0; i < columnCount; i++) { + columns[i] = decodeColumn(defs[i], rowCount); + } + + ArrayList rows = new ArrayList<>(rowCount); + for (int row = 0; row < rowCount; row++) { + LinkedHashMap values = new LinkedHashMap<>(); + for (int col = 0; col < columnCount; col++) { + values.put(defs[col].getName(), columns[col].rows[row]); + } + rows.add(new DecodedRow(tableName, values)); + } + return rows; + } + } + + private static final class DecodedRow { + private final String table; + private final LinkedHashMap values; + + private DecodedRow(String table, LinkedHashMap values) { + this.table = table; + this.values = values; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DecodedRow that = (DecodedRow) o; + return Objects.equals(table, that.table) && Objects.equals(values, that.values); + } + + @Override + public int hashCode() { + return Objects.hash(table, values); + } + + @Override + public String toString() { + return "DecodedRow{" + + "table='" + table + '\'' + + ", values=" + values + + '}'; + } + } + + private static final class DoubleArrayValue { + private final List shape; + private final List values; + + private DoubleArrayValue(List shape, List values) { + this.shape = shape; + this.values = values; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DoubleArrayValue that = (DoubleArrayValue) o; + return Objects.equals(shape, that.shape) && Objects.equals(values, that.values); + } + + @Override + public int hashCode() { + return Objects.hash(shape, values); + } + + @Override + public String toString() { + return "DoubleArrayValue{" + + "shape=" + shape + + ", values=" + values + + '}'; + } + } + + private static final class LongArrayValue { + private final List shape; + private final List values; + + private LongArrayValue(List shape, List values) { + this.shape = shape; + this.values = values; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + LongArrayValue that = (LongArrayValue) o; + return Objects.equals(shape, that.shape) && Objects.equals(values, that.values); + } + + @Override + public int hashCode() { + return Objects.hash(shape, values); + } + + @Override + public String toString() { + return "LongArrayValue{" + + "shape=" + shape + + ", values=" + values + + '}'; + } + } + + private static final class PacketReader { + private final byte[] data; + private int position; + + private PacketReader(byte[] data) { + this.data = data; + } + + private int length() { + return data.length; + } + + private int position() { + return position; + } + + private byte readByte() { + return data[position++]; + } + + private byte[] readBytes(int len) { + byte[] bytes = Arrays.copyOfRange(data, position, position + len); + position += len; + return bytes; + } + + private double readDoubleLE() { + double value = Unsafe.getUnsafe().getDouble(data, Unsafe.BYTE_OFFSET + position); + position += Double.BYTES; + return value; + } + + private int readIntLE() { + int value = Unsafe.byteArrayGetInt(data, position); + position += Integer.BYTES; + return value; + } + + private long readLongLE() { + long value = Unsafe.byteArrayGetLong(data, position); + position += Long.BYTES; + return value; + } + + private boolean[] readNullBitmap(int rowCount) { + byte[] bitmap = readBytes((rowCount + 7) / 8); + boolean[] nulls = new boolean[rowCount]; + for (int row = 0; row < rowCount; row++) { + int byteIndex = row >>> 3; + int bitIndex = row & 7; + nulls[row] = (bitmap[byteIndex] & (1 << bitIndex)) != 0; + } + return nulls; + } + + private String readString() { + int len = (int) readVarint(); + if (len == 0) { + return ""; + } + String value = new String(data, position, len, StandardCharsets.UTF_8); + position += len; + return value; + } + + private int readUnsignedByte() { + return readByte() & 0xff; + } + + private int readUnsignedShortLE() { + int value = Unsafe.byteArrayGetShort(data, position) & 0xffff; + position += Short.BYTES; + return value; + } + + private long readVarint() { + long value = 0; + int shift = 0; + while (true) { + int b = readUnsignedByte(); + value |= (long) (b & 0x7f) << shift; + if ((b & 0x80) == 0) { + return value; + } + shift += 7; + if (shift > 63) { + throw new AssertionError("varint too long"); + } + } + } + } + + private static final class RunResult { + private final List lengths; + private final List packets; + private final int sendCount; + + private RunResult(List packets, List lengths, int sendCount) { + this.packets = new ArrayList<>(packets); + this.lengths = new ArrayList<>(lengths); + this.sendCount = sendCount; + } + } + + private static final class ScenarioRow { + private final DecodedRow expected; + private final ThrowingConsumer writer; + + private ScenarioRow(DecodedRow expected, ThrowingConsumer writer) { + this.expected = expected; + this.writer = writer; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketAckIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketAckIntegrationTest.java new file mode 100644 index 0000000..81524f1 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketAckIntegrationTest.java @@ -0,0 +1,299 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; +import io.questdb.client.test.AbstractTest; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Integration tests for QWP v1 WebSocket ACK delivery mechanism. + * These tests verify that the InFlightWindow and ACK responses work correctly end-to-end. + */ +public class QwpWebSocketAckIntegrationTest extends AbstractTest { + + private static final int TEST_PORT = 19_500 + (int) (System.nanoTime() % 100); + + @Test + public void testAsyncFlushFailsFastOnInvalidAckPayload() throws Exception { + InvalidAckPayloadHandler handler = new InvalidAckPayloadHandler(); + int port = TEST_PORT + 21; + + try (TestWebSocketServer server = new TestWebSocketServer(port, handler)) { + server.start(); + Assert.assertTrue("Server failed to start", server.awaitStart(5, TimeUnit.SECONDS)); + + boolean errorCaught = false; + long start = System.currentTimeMillis(); + try (QwpWebSocketSender sender = QwpWebSocketSender.connect( + "localhost", port, false, 0, 0, 0, QwpWebSocketSender.DEFAULT_IN_FLIGHT_WINDOW_SIZE, null)) { + sender.table("test") + .longColumn("value", 1) + .atNow(); + sender.flush(); + } catch (Exception e) { + errorCaught = true; + Assert.assertTrue( + e.getMessage().contains("Invalid ACK response payload") + || e.getMessage().contains("Error in send queue") + ); + } + + long duration = System.currentTimeMillis() - start; + Assert.assertTrue("Expected invalid ACK error", errorCaught); + Assert.assertTrue("Flush should fail quickly on invalid ACK [duration=" + duration + "ms]", duration < 10_000); + } + } + + @Test + public void testAsyncFlushFailsFastOnServerClose() throws Exception { + ClosingServerHandler handler = new ClosingServerHandler(); + int port = TEST_PORT + 20; + + try (TestWebSocketServer server = new TestWebSocketServer(port, handler)) { + server.start(); + Assert.assertTrue("Server failed to start", server.awaitStart(5, TimeUnit.SECONDS)); + + boolean errorCaught = false; + long start = System.currentTimeMillis(); + try (QwpWebSocketSender sender = QwpWebSocketSender.connect( + "localhost", port, false, 0, 0, 0, QwpWebSocketSender.DEFAULT_IN_FLIGHT_WINDOW_SIZE, null)) { + sender.table("test") + .longColumn("value", 1) + .atNow(); + sender.flush(); + } catch (Exception e) { + errorCaught = true; + Assert.assertTrue( + e.getMessage().contains("closed") + || e.getMessage().contains("Error in send queue") + || e.getMessage().contains("failed") + ); + } + + long duration = System.currentTimeMillis() - start; + Assert.assertTrue("Expected async close error", errorCaught); + Assert.assertTrue("Flush should fail quickly on close [duration=" + duration + "ms]", duration < 10_000); + } + } + + /** + * Test that flush blocks until ACK is received. + * Uses async mode to enable ACK handling via InFlightWindow. + */ + @Test + public void testFlushBlocksUntilAcked() throws Exception { + final long DELAY_MS = 300; // 300ms delay before ACK + DelayedAckHandler handler = new DelayedAckHandler(DELAY_MS); + + int port = TEST_PORT + 10; + try (TestWebSocketServer server = new TestWebSocketServer(port, handler)) { + server.start(); + Assert.assertTrue("Server failed to start", server.awaitStart(5, TimeUnit.SECONDS)); + + try (QwpWebSocketSender sender = QwpWebSocketSender.connect( + "localhost", port, false, 0, 0, 0, QwpWebSocketSender.DEFAULT_IN_FLIGHT_WINDOW_SIZE, null)) { + + sender.table("test") + .longColumn("value", 42) + .atNow(); + + long startTime = System.currentTimeMillis(); + sender.flush(); + long duration = System.currentTimeMillis() - startTime; + + Assert.assertTrue("Flush should have waited for ACK (took " + duration + "ms, expected >= " + (DELAY_MS / 2) + "ms)", + duration >= DELAY_MS / 2); + + LOG.info("Flush waited {}ms for ACK", duration); + } + } + } + + @Test + public void testSyncFlushFailsOnInvalidAckPayload() throws Exception { + InvalidAckPayloadHandler handler = new InvalidAckPayloadHandler(); + int port = TEST_PORT + 22; + + try (TestWebSocketServer server = new TestWebSocketServer(port, handler)) { + server.start(); + Assert.assertTrue("Server failed to start", server.awaitStart(5, TimeUnit.SECONDS)); + + boolean errorCaught = false; + long start = System.currentTimeMillis(); + try (QwpWebSocketSender sender = QwpWebSocketSender.connect("localhost", port, false)) { + sender.table("test") + .longColumn("value", 7) + .atNow(); + sender.flush(); + } catch (Exception e) { + errorCaught = true; + Assert.assertTrue( + e.getMessage().contains("Invalid ACK response payload") + || e.getMessage().contains("Failed to parse ACK response") + ); + } + + long duration = System.currentTimeMillis() - start; + Assert.assertTrue("Expected invalid ACK error in sync mode", errorCaught); + Assert.assertTrue("Sync invalid ACK path should fail quickly [duration=" + duration + "ms]", duration < 10_000); + } + } + + @Test + public void testSyncFlushIgnoresPingAndWaitsForAck() throws Exception { + final long ackDelayMs = 300; + PingThenDelayedAckHandler handler = new PingThenDelayedAckHandler(ackDelayMs); + int port = TEST_PORT + 23; + + try (TestWebSocketServer server = new TestWebSocketServer(port, handler)) { + server.start(); + Assert.assertTrue("Server failed to start", server.awaitStart(5, TimeUnit.SECONDS)); + + try (QwpWebSocketSender sender = QwpWebSocketSender.connect("localhost", port, false)) { + sender.table("test") + .longColumn("value", 11) + .atNow(); + + long start = System.currentTimeMillis(); + sender.flush(); + long duration = System.currentTimeMillis() - start; + + Assert.assertTrue("Flush returned too early [duration=" + duration + "ms]", duration >= ackDelayMs / 2); + } + } + } + + /** + * Creates a binary ACK response using WebSocketResponse format. + * Format: status (1 byte) + sequence (8 bytes little-endian) + */ + private static byte[] createAckResponse(long sequence) { + byte[] response = new byte[WebSocketResponse.MIN_RESPONSE_SIZE]; + + // Status OK (0) + response[0] = WebSocketResponse.STATUS_OK; + + // Sequence (little-endian) + response[1] = (byte) (sequence & 0xFF); + response[2] = (byte) ((sequence >> 8) & 0xFF); + response[3] = (byte) ((sequence >> 16) & 0xFF); + response[4] = (byte) ((sequence >> 24) & 0xFF); + response[5] = (byte) ((sequence >> 32) & 0xFF); + response[6] = (byte) ((sequence >> 40) & 0xFF); + response[7] = (byte) ((sequence >> 48) & 0xFF); + response[8] = (byte) ((sequence >> 56) & 0xFF); + + return response; + } + + private static class ClosingServerHandler implements TestWebSocketServer.WebSocketServerHandler { + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + try { + client.sendClose(WebSocketCloseCode.GOING_AWAY, "bye"); + } catch (IOException e) { + LOG.error("Failed to send close frame", e); + } + } + } + + /** + * Server handler that delays ACKs to test blocking behavior. + */ + private static class DelayedAckHandler implements TestWebSocketServer.WebSocketServerHandler { + private final long delayMs; + private final AtomicLong nextSequence = new AtomicLong(0); + + DelayedAckHandler(long delayMs) { + this.delayMs = delayMs; + } + + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + long sequence = nextSequence.getAndIncrement(); + + LOG.debug("Server delaying ACK by {}ms", delayMs); + + new Thread(() -> { + try { + Thread.sleep(delayMs); + byte[] ackResponse = createAckResponse(sequence); + client.sendBinary(ackResponse); + LOG.debug("Server sent delayed ACK for seq {}", sequence); + } catch (Exception e) { + LOG.error("Failed to send delayed ACK", e); + } + }).start(); + } + } + + private static class InvalidAckPayloadHandler implements TestWebSocketServer.WebSocketServerHandler { + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + try { + client.sendBinary(new byte[]{1, 2, 3}); + } catch (IOException e) { + LOG.error("Failed to send invalid payload", e); + } + } + } + + private static class PingThenDelayedAckHandler implements TestWebSocketServer.WebSocketServerHandler { + private final long delayMs; + private final AtomicLong nextSequence = new AtomicLong(0); + + private PingThenDelayedAckHandler(long delayMs) { + this.delayMs = delayMs; + } + + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + long sequence = nextSequence.getAndIncrement(); + try { + client.sendPing(new byte[]{42}); + } catch (IOException e) { + LOG.error("Failed to send ping", e); + } + + new Thread(() -> { + try { + Thread.sleep(delayMs); + client.sendBinary(createAckResponse(sequence)); + } catch (Exception e) { + LOG.error("Failed to send delayed ACK", e); + } + }).start(); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java new file mode 100644 index 0000000..7ab65bf --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java @@ -0,0 +1,1438 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; +import io.questdb.client.cutlass.qwp.client.QwpBufferWriter; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +/** + * Unit tests for QwpWebSocketEncoder. + */ +public class QwpWebSocketEncoderTest { + + @Test + public void testBufferResetAndReuse() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test")) { + + // First batch + for (int i = 0; i < 100; i++) { + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(i); + buffer.nextRow(); + } + int size1 = encoder.encode(buffer, false); + + // Reset and second batch + buffer.reset(); + for (int i = 0; i < 50; i++) { + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(i * 2); + buffer.nextRow(); + } + int size2 = encoder.encode(buffer, false); + + Assert.assertTrue(size1 > size2); // More rows = larger + Assert.assertEquals(50, buffer.getRowCount()); + } + }); + } + + @Test + public void testEncode2DDoubleArray() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("matrix", TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(new double[][]{{1.0, 2.0}, {3.0, 4.0}}); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); + } + + @Test + public void testEncode2DLongArray() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("matrix", TYPE_LONG_ARRAY, true); + col.addLongArray(new long[][]{{1L, 2L}, {3L, 4L}}); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); + } + + @Test + public void testEncode3DDoubleArray() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("tensor", TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(new double[][][]{ + {{1.0, 2.0}, {3.0, 4.0}}, + {{5.0, 6.0}, {7.0, 8.0}} + }); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); + } + + @Test + public void testEncodeAllBasicTypesInOneRow() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("all_types")) { + + buffer.getOrCreateColumn("b", TYPE_BOOLEAN, false).addBoolean(true); + buffer.getOrCreateColumn("by", TYPE_BYTE, false).addByte((byte) 42); + buffer.getOrCreateColumn("sh", TYPE_SHORT, false).addShort((short) 1000); + buffer.getOrCreateColumn("i", TYPE_INT, false).addInt(100000); + buffer.getOrCreateColumn("l", TYPE_LONG, false).addLong(1000000000L); + buffer.getOrCreateColumn("f", TYPE_FLOAT, false).addFloat(3.14f); + buffer.getOrCreateColumn("d", TYPE_DOUBLE, false).addDouble(3.14159265); + buffer.getOrCreateColumn("s", TYPE_STRING, true).addString("test"); + buffer.getOrCreateColumn("sym", TYPE_SYMBOL, false).addSymbol("AAPL"); + buffer.getOrCreateDesignatedTimestampColumn(TYPE_TIMESTAMP).addLong(1000000L); + + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(1, buffer.getRowCount()); + } + }); + } + + @Test + public void testEncodeAllBooleanValues() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("flag", TYPE_BOOLEAN, false); + for (int i = 0; i < 100; i++) { + col.addBoolean(i % 2 == 0); + buffer.nextRow(); + } + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(100, buffer.getRowCount()); + } + }); + } + + @Test + public void testEncodeDecimal128() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("amount", TYPE_DECIMAL128, false); + col.addDecimal128(io.questdb.client.std.Decimal128.fromLong(123456789012345L, 4)); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); + } + + @Test + public void testEncodeDecimal256() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("bignum", TYPE_DECIMAL256, false); + col.addDecimal256(io.questdb.client.std.Decimal256.fromLong(Long.MAX_VALUE, 6)); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); + } + + @Test + public void testEncodeDecimal64() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("price", TYPE_DECIMAL64, false); + col.addDecimal64(io.questdb.client.std.Decimal64.fromLong(12345L, 2)); // 123.45 + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); + } + + @Test + public void testEncodeDoubleArray() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("values", TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(new double[]{1.0, 2.0, 3.0}); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); + } + + @Test + public void testEncodeEmptyString() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("name", TYPE_STRING, true); + col.addString(""); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); + } + + @Test + public void testEncodeEmptyTableName() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("")) { + // Edge case: empty table name (probably invalid but let's verify encoding works) + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(1L); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 0); + } + }); + } + + @Test + public void testEncodeLargeArray() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + // Large 1D array + double[] largeArray = new double[1000]; + for (int i = 0; i < 1000; i++) { + largeArray[i] = i * 1.5; + } + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("values", TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(largeArray); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 8000); // At least 8 bytes per double + } + }); + } + + @Test + public void testEncodeLargeRowCount() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("metrics")) { + + for (int i = 0; i < 10_000; i++) { + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(i); + buffer.nextRow(); + } + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(10_000, buffer.getRowCount()); + } + }); + } + + @Test + public void testEncodeLongArray() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("values", TYPE_LONG_ARRAY, true); + col.addLongArray(new long[]{1L, 2L, 3L}); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); + } + + // ==================== SYMBOL COLUMN TESTS ==================== + + @Test + public void testEncodeLongString() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + String sb = "a".repeat(10_000); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("data", TYPE_STRING, true); + col.addString(sb); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 10_000); + } + }); + } + + @Test + public void testEncodeMaxMinLong() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(Long.MAX_VALUE); + buffer.nextRow(); + + col.addLong(Long.MIN_VALUE); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(2, buffer.getRowCount()); + } + }); + } + + @Test + public void testEncodeMixedColumnTypes() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("events")) { + + // Add columns of different types + QwpTableBuffer.ColumnBuffer symbolCol = buffer.getOrCreateColumn("host", TYPE_SYMBOL, false); + symbolCol.addSymbol("server1"); + + QwpTableBuffer.ColumnBuffer longCol = buffer.getOrCreateColumn("count", TYPE_LONG, false); + longCol.addLong(42); + + QwpTableBuffer.ColumnBuffer doubleCol = buffer.getOrCreateColumn("value", TYPE_DOUBLE, false); + doubleCol.addDouble(3.14); + + QwpTableBuffer.ColumnBuffer boolCol = buffer.getOrCreateColumn("active", TYPE_BOOLEAN, false); + boolCol.addBoolean(true); + + QwpTableBuffer.ColumnBuffer stringCol = buffer.getOrCreateColumn("message", TYPE_STRING, true); + stringCol.addString("hello world"); + + QwpTableBuffer.ColumnBuffer tsCol = buffer.getOrCreateDesignatedTimestampColumn(TYPE_TIMESTAMP); + tsCol.addLong(1000000L); + + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); + } + + @Test + public void testEncodeMixedColumnsMultipleRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("events")) { + + for (int i = 0; i < 50; i++) { + QwpTableBuffer.ColumnBuffer symbolCol = buffer.getOrCreateColumn("host", TYPE_SYMBOL, false); + symbolCol.addSymbol("server" + (i % 5)); + + QwpTableBuffer.ColumnBuffer longCol = buffer.getOrCreateColumn("count", TYPE_LONG, false); + longCol.addLong(i * 10); + + QwpTableBuffer.ColumnBuffer doubleCol = buffer.getOrCreateColumn("value", TYPE_DOUBLE, false); + doubleCol.addDouble(i * 1.5); + + QwpTableBuffer.ColumnBuffer tsCol = buffer.getOrCreateDesignatedTimestampColumn(TYPE_TIMESTAMP); + tsCol.addLong(1000000L + i); + + buffer.nextRow(); + } + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(50, buffer.getRowCount()); + } + }); + } + + // ==================== UUID COLUMN TESTS ==================== + + @Test + public void testEncodeMultipleColumns() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("weather")) { + + // Add multiple columns + QwpTableBuffer.ColumnBuffer tempCol = buffer.getOrCreateColumn("temperature", TYPE_DOUBLE, false); + tempCol.addDouble(23.5); + + QwpTableBuffer.ColumnBuffer humCol = buffer.getOrCreateColumn("humidity", TYPE_LONG, false); + humCol.addLong(65); + + QwpTableBuffer.ColumnBuffer tsCol = buffer.getOrCreateDesignatedTimestampColumn(TYPE_TIMESTAMP); + tsCol.addLong(1000000L); + + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + + // Verify header + QwpBufferWriter buf = encoder.getBuffer(); + long ptr = buf.getBufferPtr(); + Assert.assertEquals((byte) 'Q', Unsafe.getUnsafe().getByte(ptr)); + Assert.assertEquals((byte) 'W', Unsafe.getUnsafe().getByte(ptr + 1)); + Assert.assertEquals((byte) 'P', Unsafe.getUnsafe().getByte(ptr + 2)); + Assert.assertEquals((byte) '1', Unsafe.getUnsafe().getByte(ptr + 3)); + } + }); + } + + @Test + public void testEncodeMultipleDecimal64() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("price", TYPE_DECIMAL64, false); + col.addDecimal64(io.questdb.client.std.Decimal64.fromLong(12345L, 2)); // 123.45 + buffer.nextRow(); + + col.addDecimal64(io.questdb.client.std.Decimal64.fromLong(67890L, 2)); // 678.90 + buffer.nextRow(); + + col.addDecimal64(io.questdb.client.std.Decimal64.fromLong(11111L, 2)); // 111.11 + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(3, buffer.getRowCount()); + } + }); + } + + // ==================== DECIMAL COLUMN TESTS ==================== + + @Test + public void testEncodeMultipleRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("metrics")) { + + for (int i = 0; i < 100; i++) { + QwpTableBuffer.ColumnBuffer valCol = buffer.getOrCreateColumn("value", TYPE_LONG, false); + valCol.addLong(i); + + QwpTableBuffer.ColumnBuffer tsCol = buffer.getOrCreateDesignatedTimestampColumn(TYPE_TIMESTAMP); + tsCol.addLong(1000000L + i); + + buffer.nextRow(); + } + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(100, buffer.getRowCount()); + } + }); + } + + @Test + public void testEncodeMultipleSymbolsSameDictionary() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("host", TYPE_SYMBOL, false); + col.addSymbol("server1"); + buffer.nextRow(); + + col.addSymbol("server1"); // Same symbol + buffer.nextRow(); + + col.addSymbol("server2"); // Different symbol + buffer.nextRow(); + + col.addSymbol("server1"); // Back to first + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(4, buffer.getRowCount()); + } + }); + } + + @Test + public void testEncodeMultipleUuids() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("id", TYPE_UUID, false); + for (int i = 0; i < 10; i++) { + col.addUuid(i * 1000L, i * 2000L); + buffer.nextRow(); + } + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(10, buffer.getRowCount()); + } + }); + } + + @Test + public void testEncodeNaNDouble() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_DOUBLE, false); + col.addDouble(Double.NaN); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); + } + + // ==================== ARRAY COLUMN TESTS ==================== + + @Test + public void testEncodeNegativeLong() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(-123456789L); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); + } + + @Test + public void testEncodeNullableColumnWithNull() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test")) { + + // Nullable column with null + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("name", TYPE_STRING, true); + col.addString(null); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); + } + + @Test + public void testEncodeNullableColumnWithValue() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test")) { + + // Nullable column with a value + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("name", TYPE_STRING, true); + col.addString("hello"); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); + } + + @Test + public void testEncodeNullableSymbolWithNull() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("host", TYPE_SYMBOL, true); + col.addSymbol("server1"); + buffer.nextRow(); + + col.addSymbol(null); // Null symbol + buffer.nextRow(); + + col.addSymbol("server2"); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(3, buffer.getRowCount()); + } + }); + } + + // ==================== MULTIPLE ROWS TESTS ==================== + + @Test + public void testEncodeSingleRowWithBoolean() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("active", TYPE_BOOLEAN, false); + col.addBoolean(true); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); + } + + @Test + public void testEncodeSingleRowWithDouble() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("temperature", TYPE_DOUBLE, false); + col.addDouble(23.5); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); + } + + // ==================== MIXED COLUMN TYPES ==================== + + @Test + public void testEncodeSingleRowWithLong() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + // Add a long column + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("value", TYPE_LONG, false); + col.addLong(12345L); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); // At least header size + + QwpBufferWriter buf = encoder.getBuffer(); + long ptr = buf.getBufferPtr(); + + // Verify header magic + Assert.assertEquals((byte) 'Q', Unsafe.getUnsafe().getByte(ptr)); + Assert.assertEquals((byte) 'W', Unsafe.getUnsafe().getByte(ptr + 1)); + Assert.assertEquals((byte) 'P', Unsafe.getUnsafe().getByte(ptr + 2)); + Assert.assertEquals((byte) '1', Unsafe.getUnsafe().getByte(ptr + 3)); + + // Version + Assert.assertEquals(VERSION_1, Unsafe.getUnsafe().getByte(ptr + 4)); + + // Table count (little-endian short) + Assert.assertEquals((short) 1, Unsafe.getUnsafe().getShort(ptr + 6)); + } + }); + } + + @Test + public void testEncodeSingleRowWithString() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("name", TYPE_STRING, true); + col.addString("hello"); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); + } + + // ==================== EDGE CASES ==================== + + @Test + public void testEncodeSingleRowWithTimestamp() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + // Add a timestamp column (designated timestamp uses empty name) + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateDesignatedTimestampColumn(TYPE_TIMESTAMP); + col.addLong(1000000L); // Micros + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); + } + + @Test + public void testEncodeSingleSymbol() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("host", TYPE_SYMBOL, false); + col.addSymbol("server1"); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); + } + + @Test + public void testEncodeSpecialDoubles() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_DOUBLE, false); + col.addDouble(Double.MAX_VALUE); + buffer.nextRow(); + + col.addDouble(Double.MIN_VALUE); + buffer.nextRow(); + + col.addDouble(Double.POSITIVE_INFINITY); + buffer.nextRow(); + + col.addDouble(Double.NEGATIVE_INFINITY); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(4, buffer.getRowCount()); + } + }); + } + + @Test + public void testEncodeSymbolWithManyDistinctValues() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("host", TYPE_SYMBOL, false); + for (int i = 0; i < 100; i++) { + col.addSymbol("server" + i); + buffer.nextRow(); + } + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(100, buffer.getRowCount()); + } + }); + } + + @Test + public void testEncodeUnicodeString() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("name", TYPE_STRING, true); + col.addString("Hello 世界 🌍"); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); + } + + @Test + public void testEncodeUuid() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("id", TYPE_UUID, false); + col.addUuid(0x123456789ABCDEF0L, 0xFEDCBA9876543210L); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); + } + + @Test + public void testEncodeWithDeltaDict_freshConnection_sendsAllSymbols() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Add symbol column with global IDs + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ticker", TYPE_SYMBOL, false); + + // Simulate adding symbols via global dictionary + int id1 = globalDict.getOrAddSymbol("AAPL"); // ID 0 + int id2 = globalDict.getOrAddSymbol("GOOG"); // ID 1 + col.addSymbolWithGlobalId("AAPL", id1); + buffer.nextRow(); + col.addSymbolWithGlobalId("GOOG", id2); + buffer.nextRow(); + + // Fresh connection: confirmedMaxId = -1, so delta should include all symbols (0, 1) + int confirmedMaxId = -1; + int batchMaxId = 1; + + int size = encoder.encodeWithDeltaDict(buffer, globalDict, confirmedMaxId, batchMaxId, false); + Assert.assertTrue(size > 12); + + QwpBufferWriter buf = encoder.getBuffer(); + long ptr = buf.getBufferPtr(); + + // Verify header flag has FLAG_DELTA_SYMBOL_DICT set + byte flags = Unsafe.getUnsafe().getByte(ptr + HEADER_OFFSET_FLAGS); + Assert.assertTrue("Delta flag should be set", (flags & FLAG_DELTA_SYMBOL_DICT) != 0); + } + }); + } + + @Test + public void testEncodeWithDeltaDict_noNewSymbols_sendsEmptyDelta() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Pre-populate dictionary with all symbols + int id0 = globalDict.getOrAddSymbol("AAPL"); // ID 0 + int id1 = globalDict.getOrAddSymbol("GOOG"); // ID 1 + + // Use only existing symbols + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ticker", TYPE_SYMBOL, false); + col.addSymbolWithGlobalId("AAPL", id0); + buffer.nextRow(); + col.addSymbolWithGlobalId("GOOG", id1); + buffer.nextRow(); + + // Server has confirmed all symbols (0-1), batchMaxId is 1 + int confirmedMaxId = 1; + int batchMaxId = 1; + + int size = encoder.encodeWithDeltaDict(buffer, globalDict, confirmedMaxId, batchMaxId, false); + Assert.assertTrue(size > 12); + + QwpBufferWriter buf = encoder.getBuffer(); + long ptr = buf.getBufferPtr(); + + // Verify delta flag is set + byte flags = Unsafe.getUnsafe().getByte(ptr + HEADER_OFFSET_FLAGS); + Assert.assertTrue("Delta flag should be set", (flags & FLAG_DELTA_SYMBOL_DICT) != 0); + + // Read delta section after header + long pos = ptr + HEADER_SIZE; + + // Read deltaStart varint (should be 2 = confirmedMaxId + 1) + int deltaStart = Unsafe.getUnsafe().getByte(pos) & 0x7F; + Assert.assertEquals(2, deltaStart); + pos++; + + // Read deltaCount varint (should be 0) + int deltaCount = Unsafe.getUnsafe().getByte(pos) & 0x7F; + Assert.assertEquals(0, deltaCount); + } + }); + } + + @Test + public void testEncodeWithDeltaDict_withConfirmed_sendsOnlyNew() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Pre-populate dictionary (simulating symbols already sent) + globalDict.getOrAddSymbol("AAPL"); // ID 0 + globalDict.getOrAddSymbol("GOOG"); // ID 1 + + // Now add new symbols + int id2 = globalDict.getOrAddSymbol("MSFT"); // ID 2 + int id3 = globalDict.getOrAddSymbol("TSLA"); // ID 3 + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ticker", TYPE_SYMBOL, false); + col.addSymbolWithGlobalId("MSFT", id2); + buffer.nextRow(); + col.addSymbolWithGlobalId("TSLA", id3); + buffer.nextRow(); + + // Server has confirmed IDs 0-1, so delta should only include 2-3 + int confirmedMaxId = 1; + int batchMaxId = 3; + + int size = encoder.encodeWithDeltaDict(buffer, globalDict, confirmedMaxId, batchMaxId, false); + Assert.assertTrue(size > 12); + + QwpBufferWriter buf = encoder.getBuffer(); + long ptr = buf.getBufferPtr(); + + // Verify delta flag is set + byte flags = Unsafe.getUnsafe().getByte(ptr + HEADER_OFFSET_FLAGS); + Assert.assertTrue("Delta flag should be set", (flags & FLAG_DELTA_SYMBOL_DICT) != 0); + + // Read delta section after header + long pos = ptr + HEADER_SIZE; + + // Read deltaStart varint (should be 2 = confirmedMaxId + 1) + int deltaStart = Unsafe.getUnsafe().getByte(pos) & 0x7F; + Assert.assertEquals(2, deltaStart); + } + }); + } + + @Test + public void testEncodeWithDeltaDict_readsGlobalIdsFromDataBuffer() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + for (int i = 0; i < 8; i++) { + globalDict.getOrAddSymbol("SYM_" + i); + } + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ticker", TYPE_SYMBOL, false); + col.addSymbolWithGlobalId("SYM_5", 5); + buffer.nextRow(); + col.addSymbolWithGlobalId("SYM_7", 7); + buffer.nextRow(); + + Assert.assertEquals(0, col.getAuxDataAddress()); + + int size = encoder.encodeWithDeltaDict(buffer, globalDict, 7, 7, false); + Assert.assertTrue(size > 12); + + Cursor cursor = new Cursor(encoder.getBuffer().getBufferPtr() + HEADER_SIZE); + Assert.assertEquals(8, cursor.readVarint()); + Assert.assertEquals(0, cursor.readVarint()); + + Assert.assertEquals("test_table", cursor.readString()); + Assert.assertEquals(2, cursor.readVarint()); + Assert.assertEquals(1, cursor.readVarint()); + Assert.assertEquals(SCHEMA_MODE_FULL, cursor.readByte()); + Assert.assertEquals("ticker", cursor.readString()); + Assert.assertEquals(TYPE_SYMBOL, cursor.readByte()); + Assert.assertEquals(0, cursor.readByte()); // no nulls + Assert.assertEquals(5, cursor.readVarint()); + Assert.assertEquals(7, cursor.readVarint()); + } + }); + } + + // ==================== SCHEMA REFERENCE TESTS ==================== + + @Test + public void testEncodeWithSchemaRef() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(42L); + buffer.nextRow(); + + int size = encoder.encode(buffer, true); // Use schema reference + Assert.assertTrue(size > 12); + } + }); + } + + // ==================== BUFFER REUSE TESTS ==================== + + @Test + public void testEncodeZeroLong() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(0L); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); + } + + @Test + public void testEncoderReusability() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer1 = new QwpTableBuffer("table1"); + QwpTableBuffer buffer2 = new QwpTableBuffer("table2")) { + // Encode first message + QwpTableBuffer.ColumnBuffer col1 = buffer1.getOrCreateColumn("x", TYPE_LONG, false); + col1.addLong(1L); + buffer1.nextRow(); + int size1 = encoder.encode(buffer1, false); + + // Encode second message (encoder should reset internally) + QwpTableBuffer.ColumnBuffer col2 = buffer2.getOrCreateColumn("y", TYPE_DOUBLE, false); + col2.addDouble(2.0); + buffer2.nextRow(); + int size2 = encoder.encode(buffer2, false); + + // Both should succeed + Assert.assertTrue(size1 > 12); + Assert.assertTrue(size2 > 12); + } + }); + } + + // ==================== ALL BASIC TYPES IN ONE ROW ==================== + + @Test + public void testGlobalSymbolDictionaryBasics() throws Exception { + assertMemoryLeak(() -> { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + // Test sequential IDs + Assert.assertEquals(0, dict.getOrAddSymbol("AAPL")); + Assert.assertEquals(1, dict.getOrAddSymbol("GOOG")); + Assert.assertEquals(2, dict.getOrAddSymbol("MSFT")); + + // Test deduplication + Assert.assertEquals(0, dict.getOrAddSymbol("AAPL")); + Assert.assertEquals(1, dict.getOrAddSymbol("GOOG")); + + // Test retrieval + Assert.assertEquals("AAPL", dict.getSymbol(0)); + Assert.assertEquals("GOOG", dict.getSymbol(1)); + Assert.assertEquals("MSFT", dict.getSymbol(2)); + + // Test size + Assert.assertEquals(3, dict.size()); + }); + } + + // ==================== Delta Symbol Dictionary Tests ==================== + + @Test + public void testGorillaEncoding_compressionRatio() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("metrics")) { + encoder.setGorillaEnabled(true); + + // Add many timestamps with constant delta - best case for Gorilla + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ts", TYPE_TIMESTAMP, true); + for (int i = 0; i < 1000; i++) { + col.addLong(1000000000L + i * 1000L); + buffer.nextRow(); + } + + int sizeWithGorilla = encoder.encode(buffer, false); + + // Calculate theoretical minimum size for Gorilla: + // - Header: 12 bytes + // - Table header, column schema, etc. + // - First timestamp: 8 bytes + // - Second timestamp: 8 bytes + // - Remaining 998 timestamps: 998 bits (1 bit each for DoD=0) = ~125 bytes + + // Calculate size without Gorilla (1000 * 8 = 8000 bytes just for timestamps) + encoder.setGorillaEnabled(false); + buffer.reset(); + col = buffer.getOrCreateColumn("ts", TYPE_TIMESTAMP, true); + for (int i = 0; i < 1000; i++) { + col.addLong(1000000000L + i * 1000L); + buffer.nextRow(); + } + + int sizeWithoutGorilla = encoder.encode(buffer, false); + + // For constant delta, Gorilla should achieve significant compression + double compressionRatio = (double) sizeWithGorilla / sizeWithoutGorilla; + Assert.assertTrue("Compression ratio should be < 0.2 for constant delta", + compressionRatio < 0.2); + } + }); + } + + @Test + public void testGorillaEncoding_multipleTimestampColumns() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test")) { + encoder.setGorillaEnabled(true); + + // Add multiple timestamp columns + for (int i = 0; i < 50; i++) { + QwpTableBuffer.ColumnBuffer ts1Col = buffer.getOrCreateColumn("ts1", TYPE_TIMESTAMP, true); + ts1Col.addLong(1000000000L + i * 1000L); + + QwpTableBuffer.ColumnBuffer ts2Col = buffer.getOrCreateColumn("ts2", TYPE_TIMESTAMP, true); + ts2Col.addLong(2000000000L + i * 2000L); + + buffer.nextRow(); + } + + int sizeWithGorilla = encoder.encode(buffer, false); + + // Compare with uncompressed + encoder.setGorillaEnabled(false); + buffer.reset(); + for (int i = 0; i < 50; i++) { + QwpTableBuffer.ColumnBuffer ts1Col = buffer.getOrCreateColumn("ts1", TYPE_TIMESTAMP, true); + ts1Col.addLong(1000000000L + i * 1000L); + + QwpTableBuffer.ColumnBuffer ts2Col = buffer.getOrCreateColumn("ts2", TYPE_TIMESTAMP, true); + ts2Col.addLong(2000000000L + i * 2000L); + + buffer.nextRow(); + } + + int sizeWithoutGorilla = encoder.encode(buffer, false); + + Assert.assertTrue("Gorilla should compress multiple timestamp columns", + sizeWithGorilla < sizeWithoutGorilla); + } + }); + } + + @Test + public void testGorillaEncoding_multipleTimestamps_usesGorillaEncoding() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test")) { + encoder.setGorillaEnabled(true); + + // Add multiple timestamps with constant delta (best compression) + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateDesignatedTimestampColumn(TYPE_TIMESTAMP); + for (int i = 0; i < 100; i++) { + col.addLong(1000000000L + i * 1000L); + buffer.nextRow(); + } + + int sizeWithGorilla = encoder.encode(buffer, false); + + // Now encode without Gorilla + encoder.setGorillaEnabled(false); + buffer.reset(); + col = buffer.getOrCreateDesignatedTimestampColumn(TYPE_TIMESTAMP); + for (int i = 0; i < 100; i++) { + col.addLong(1000000000L + i * 1000L); + buffer.nextRow(); + } + + int sizeWithoutGorilla = encoder.encode(buffer, false); + + // Gorilla should produce smaller output for constant-delta timestamps + Assert.assertTrue("Gorilla encoding should be smaller", + sizeWithGorilla < sizeWithoutGorilla); + } + }); + } + + @Test + public void testGorillaEncoding_nanosTimestamps() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test")) { + encoder.setGorillaEnabled(true); + + // Use TYPE_TIMESTAMP_NANOS + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ts", TYPE_TIMESTAMP_NANOS, true); + for (int i = 0; i < 100; i++) { + col.addLong(1000000000000000000L + i * 1000000L); // Nanos with millisecond intervals + buffer.nextRow(); + } + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + + // Verify header has Gorilla flag + QwpBufferWriter buf = encoder.getBuffer(); + byte flags = Unsafe.getUnsafe().getByte(buf.getBufferPtr() + HEADER_OFFSET_FLAGS); + Assert.assertEquals(FLAG_GORILLA, (byte) (flags & FLAG_GORILLA)); + } + }); + } + + @Test + public void testGorillaEncoding_singleTimestamp_usesUncompressed() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test")) { + encoder.setGorillaEnabled(true); + + // Single timestamp - should use uncompressed + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateDesignatedTimestampColumn(TYPE_TIMESTAMP); + col.addLong(1000000L); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); + } + + @Test + public void testGorillaEncoding_twoTimestamps_usesUncompressed() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test")) { + encoder.setGorillaEnabled(true); + + // Only 2 timestamps - should use uncompressed (Gorilla needs 3+) + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateDesignatedTimestampColumn(TYPE_TIMESTAMP); + col.addLong(1000000L); + buffer.nextRow(); + col.addLong(2000000L); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + + // Verify header has Gorilla flag set + QwpBufferWriter buf = encoder.getBuffer(); + byte flags = Unsafe.getUnsafe().getByte(buf.getBufferPtr() + HEADER_OFFSET_FLAGS); + Assert.assertEquals(FLAG_GORILLA, (byte) (flags & FLAG_GORILLA)); + } + }); + } + + @Test + public void testGorillaEncoding_varyingDelta() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test")) { + encoder.setGorillaEnabled(true); + + // Varying deltas that exercise different buckets + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ts", TYPE_TIMESTAMP, true); + long[] timestamps = { + 1000000000L, + 1000001000L, // delta=1000 + 1000002000L, // DoD=0 + 1000003050L, // DoD=50 + 1000004200L, // DoD=100 + 1000006200L, // DoD=850 + }; + + for (long ts : timestamps) { + col.addLong(ts); + buffer.nextRow(); + } + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + + // Verify header has Gorilla flag + QwpBufferWriter buf = encoder.getBuffer(); + byte flags = Unsafe.getUnsafe().getByte(buf.getBufferPtr() + HEADER_OFFSET_FLAGS); + Assert.assertEquals(FLAG_GORILLA, (byte) (flags & FLAG_GORILLA)); + } + }); + } + + @Test + public void testGorillaFlagDisabled() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test")) { + encoder.setGorillaEnabled(false); + Assert.assertFalse(encoder.isGorillaEnabled()); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ts", TYPE_TIMESTAMP, true); + col.addLong(1000000L); + buffer.nextRow(); + + encoder.encode(buffer, false); + + // Check flags byte doesn't have Gorilla bit set + QwpBufferWriter buf = encoder.getBuffer(); + byte flags = Unsafe.getUnsafe().getByte(buf.getBufferPtr() + 5); + Assert.assertEquals(0, flags & FLAG_GORILLA); + } + }); + } + + @Test + public void testGorillaFlagEnabled() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test")) { + encoder.setGorillaEnabled(true); + Assert.assertTrue(encoder.isGorillaEnabled()); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ts", TYPE_TIMESTAMP, true); + col.addLong(1000000L); + buffer.nextRow(); + + encoder.encode(buffer, false); + + // Check flags byte has Gorilla bit set + QwpBufferWriter buf = encoder.getBuffer(); + byte flags = Unsafe.getUnsafe().getByte(buf.getBufferPtr() + 5); + Assert.assertEquals(FLAG_GORILLA, (byte) (flags & FLAG_GORILLA)); + } + }); + } + + @Test + public void testPayloadLengthPatched() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(42L); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + + // Payload length is at offset 8 (4 magic + 1 version + 1 flags + 2 tablecount) + QwpBufferWriter buf = encoder.getBuffer(); + int payloadLength = Unsafe.getUnsafe().getInt(buf.getBufferPtr() + 8); + + // Payload length should be total size minus header (12 bytes) + Assert.assertEquals(size - 12, payloadLength); + } + }); + } + + @Test + public void testReset() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(1L); + buffer.nextRow(); + + int size1 = encoder.encode(buffer, false); + + // Reset and encode again + buffer.reset(); + col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(2L); + buffer.nextRow(); + + int size2 = encoder.encode(buffer, false); + + // Sizes should be similar (same schema) + Assert.assertEquals(size1, size2); + } + }); + } + + private static final class Cursor { + private long address; + + private Cursor(long address) { + this.address = address; + } + + private byte readByte() { + return Unsafe.getUnsafe().getByte(address++); + } + + private String readString() { + int len = readVarint(); + byte[] bytes = new byte[len]; + for (int i = 0; i < len; i++) { + bytes[i] = Unsafe.getUnsafe().getByte(address + i); + } + String value = new String(bytes, StandardCharsets.UTF_8); + address += len; + return value; + } + + private int readVarint() { + int value = 0; + int shift = 0; + while (true) { + int b = Unsafe.getUnsafe().getByte(address++) & 0xff; + value |= (b & 0x7f) << shift; + if ((b & 0x80) == 0) { + return value; + } + shift += 7; + } + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java new file mode 100644 index 0000000..f96d2cf --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java @@ -0,0 +1,254 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.InFlightWindow; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.test.AbstractTest; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.time.temporal.ChronoUnit; + +/** + * Verifies {@link QwpWebSocketSender} internal state management: + *

      + *
    • {@code reset()} discards all pending state, not just the current table buffer.
      • + *
    • Cached timestamp column references are invalidated during flush operations, + * preventing stale writes through freed {@code ColumnBuffer} instances.
      • + *
    • Auto-flush accumulates rows globally across all tables rather than flushing + * per-table on each table switch.
      • + *
    + */ +public class QwpWebSocketSenderStateTest extends AbstractTest { + + @Test + public void testAutoFlushAccumulatesRowsAcrossAllTables() throws Exception { + assertMemoryLeak(() -> { + // autoFlushRows=5; bytes and interval are disabled to isolate the row-count check. + // The test verifies that switching tables does NOT trigger a flush — flush fires + // only when the TOTAL pending-row count reaches the configured threshold. + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( + "localhost", 0, 5, 0, 0L, 1 + ); + try { + setField(sender, "connected", true); + setField(sender, "inFlightWindow", new InFlightWindow(1, InFlightWindow.DEFAULT_TIMEOUT_MS)); + + // Write 4 rows interleaved between t1 and t2. + // None of these should trigger auto-flush (4 < 5 = autoFlushRows). + sender.table("t1").longColumn("x", 1).at(1, ChronoUnit.MICROS); + sender.table("t2").longColumn("y", 1).at(1, ChronoUnit.MICROS); + sender.table("t1").longColumn("x", 2).at(2, ChronoUnit.MICROS); + sender.table("t2").longColumn("y", 2).at(2, ChronoUnit.MICROS); + + // All 4 rows must still be buffered — switching tables must not flush. + QwpTableBuffer t1 = sender.getTableBuffer("t1"); + QwpTableBuffer t2 = sender.getTableBuffer("t2"); + Assert.assertEquals("t1 should have 2 buffered rows (no premature flush)", + 2, t1.getRowCount()); + Assert.assertEquals("t2 should have 2 buffered rows (no premature flush)", + 2, t2.getRowCount()); + Assert.assertEquals("pendingRowCount must reflect all 4 rows across both tables", + 4, sender.getPendingRowCount()); + + // The 5th row hits the global threshold and triggers auto-flush. + // The flush fails because client is null, confirming that flush + // was triggered by the row-count threshold, not by the table switch. + boolean flushTriggered = false; + try { + sender.table("t1").longColumn("x", 3).at(3, ChronoUnit.MICROS); + } catch (Exception expected) { + flushTriggered = true; + } + Assert.assertTrue("auto-flush must be triggered on the 5th row", flushTriggered); + } finally { + setField(sender, "connected", false); + sender.close(); + } + }); + } + + @Test + public void testCachedTimestampColumnInvalidatedDuringFlush() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( + "localhost", 0, 1, 10_000_000, 0, 1 + ); + try { + setField(sender, "connected", true); + + // Row 1: caches cachedTimestampColumn, then auto-flush + // triggers and fails (no real connection). + try { + sender.table("t") + .longColumn("x", 1) + .at(1, ChronoUnit.MICROS); + } catch (Exception ignored) { + } + + // Clear the table buffer so a stale cached reference now + // points to a freed ColumnBuffer. + QwpTableBuffer tb = sender.getTableBuffer("t"); + tb.clear(); + + // Row 2: with the fix, atMicros() creates a fresh column + // and the row is buffered. Without, addLong() NPEs before + // sendRow()/nextRow() and the row is never counted. + try { + sender.table("t") + .longColumn("x", 2) + .at(2, ChronoUnit.MICROS); + } catch (Exception ignored) { + } + + Assert.assertEquals("row must be buffered when cache is properly invalidated", + 1, tb.getRowCount()); + } finally { + setField(sender, "connected", false); + sender.close(); + } + }); + } + + @Test + public void testCachedTimestampNanosColumnInvalidatedDuringFlush() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( + "localhost", 0, 1, 10_000_000, 0, 1 + ); + try { + setField(sender, "connected", true); + + try { + sender.table("t") + .longColumn("x", 1) + .at(1, ChronoUnit.NANOS); + } catch (Exception ignored) { + } + + QwpTableBuffer tb = sender.getTableBuffer("t"); + tb.clear(); + + try { + sender.table("t") + .longColumn("x", 2) + .at(2, ChronoUnit.NANOS); + } catch (Exception ignored) { + } + + Assert.assertEquals("row must be buffered when cache is properly invalidated", + 1, tb.getRowCount()); + } finally { + setField(sender, "connected", false); + sender.close(); + } + }); + } + + @Test + public void testResetClearsAllTableBuffersAndPendingRowCount() throws Exception { + assertMemoryLeak(() -> { + // Use high autoFlushRows to prevent auto-flush during the test + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( + "localhost", 0, 10_000, 10_000_000, 0, 1 + ); + try { + // Bypass ensureConnected() — mark as connected, leave client null + setField(sender, "connected", true); + setField(sender, "inFlightWindow", new InFlightWindow(1, InFlightWindow.DEFAULT_TIMEOUT_MS)); + + // Buffer rows into two different tables via the fluent API + sender.table("t1") + .longColumn("x", 1) + .at(1, ChronoUnit.MICROS); + sender.table("t2") + .longColumn("y", 2) + .at(2, ChronoUnit.MICROS); + + // Verify data is buffered + QwpTableBuffer t1 = sender.getTableBuffer("t1"); + QwpTableBuffer t2 = sender.getTableBuffer("t2"); + Assert.assertEquals("t1 should have 1 row before reset", 1, t1.getRowCount()); + Assert.assertEquals("t2 should have 1 row before reset", 1, t2.getRowCount()); + Assert.assertEquals("pendingRowCount should be 2 before reset", 2, sender.getPendingRowCount()); + + // Select t1 as the current table + sender.table("t1"); + + // Call reset — per the Sender contract this should discard + // ALL pending state, not just the current table + sender.reset(); + + // Both table buffers should be cleared + Assert.assertEquals("t1 row count should be 0 after reset", 0, t1.getRowCount()); + Assert.assertEquals("t2 row count should be 0 after reset", 0, t2.getRowCount()); + + // Pending row count should be zeroed + Assert.assertEquals("pendingRowCount should be 0 after reset", 0, sender.getPendingRowCount()); + } finally { + setField(sender, "connected", false); + sender.close(); + } + }); + } + + @Test + public void testTimestampOnlyRows() throws Exception { + assertMemoryLeak(() -> { + // autoFlushRows=10_000 prevents auto-flush; bytes and interval disabled + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( + "localhost", 0, 10_000, 0, 0L, 1 + ); + try { + setField(sender, "connected", true); + setField(sender, "inFlightWindow", new InFlightWindow(1, InFlightWindow.DEFAULT_TIMEOUT_MS)); + + // at(micros) with no other columns + sender.table("t").at(1_000L, ChronoUnit.MICROS); + // atNow() with no other columns + sender.table("t").atNow(); + + QwpTableBuffer tb = sender.getTableBuffer("t"); + Assert.assertEquals( + "at() and atNow() with no other columns must each buffer a row", + 2, tb.getRowCount() + ); + } finally { + setField(sender, "connected", false); + sender.close(); + } + }); + } + + private static void setField(Object target, String fieldName, Object value) throws Exception { + Field f = target.getClass().getDeclaredField(fieldName); + f.setAccessible(true); + f.set(target, value); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java new file mode 100644 index 0000000..f444918 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java @@ -0,0 +1,535 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.MicrobatchBuffer; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.client.WebSocketSendQueue; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.network.PlainSocketFactory; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +/** + * Unit tests for QwpWebSocketSender. + * These tests focus on state management and API validation without requiring a live server. + */ +public class QwpWebSocketSenderTest { + + @Test + public void testAtAfterCloseThrows() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.at(1000L, ChronoUnit.MICROS); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + }); + } + + @Test + public void testAtInstantAfterCloseThrows() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.at(Instant.now()); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + }); + } + + @Test + public void testAtNowAfterCloseThrows() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.atNow(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + }); + } + + @Test + public void testBoolColumnAfterCloseThrows() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.boolColumn("x", true); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + }); + } + + @Test + public void testBufferViewNotSupported() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketSender sender = createUnconnectedSender()) { + sender.bufferView(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("not supported")); + } + }); + } + + @Test + public void testCancelRowAfterCloseThrows() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.cancelRow(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + }); + } + + @Test + public void testCancelRowDiscardsPartialRow() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketSender sender = createUnconnectedSender()) { + sender.table("test"); + sender.longColumn("x", 1); + sender.boolColumn("y", true); + + // Row is not yet committed (no at/atNow call), cancel it + sender.cancelRow(); + + // Buffer should have no committed rows + QwpTableBuffer buf = sender.getTableBuffer("test"); + Assert.assertEquals(0, buf.getRowCount()); + } + }); + } + + @Test + public void testCancelRowNoOpWithoutTable() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketSender sender = createUnconnectedSender()) { + // cancelRow without table() should be a no-op (no NPE) + sender.cancelRow(); + } + }); + } + + @Test + public void testCloseIdemponent() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + sender.close(); // Should not throw + }); + } + + @Test + public void testConnectToClosedPort() throws Exception { + assertMemoryLeak(() -> { + try { + QwpWebSocketSender.connect("127.0.0.1", 1); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("Failed to connect")); + } + }); + } + + @Test + public void testDoubleArrayAfterCloseThrows() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.doubleArray("x", new double[]{1.0, 2.0}); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + }); + } + + @Test + public void testDoubleColumnAfterCloseThrows() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.doubleColumn("x", 1.0); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + }); + } + + @Test + public void testGorillaEnabledByDefault() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketSender sender = createUnconnectedSender()) { + Assert.assertTrue(sender.isGorillaEnabled()); + } + }); + } + + @Test + public void testLongArrayAfterCloseThrows() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.longArray("x", new long[]{1L, 2L}); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + }); + } + + @Test + public void testLongColumnAfterCloseThrows() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.longColumn("x", 1); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + }); + } + + @Test + public void testNullArrayReturnsThis() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketSender sender = createUnconnectedSender()) { + // Null arrays should be no-ops and return sender + Assert.assertSame(sender, sender.doubleArray("x", (double[]) null)); + Assert.assertSame(sender, sender.longArray("x", (long[]) null)); + } + }); + } + + @Test + public void testOperationsAfterCloseThrow() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.table("test"); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + }); + } + + @Test + public void testResetAfterCloseThrows() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.reset(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + }); + } + + @Test + public void testSealAndSwapRollsBackOnEnqueueFailure() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketSender sender = createUnconnectedAsyncSender(); ThrowingOnceWebSocketSendQueue queue = new ThrowingOnceWebSocketSendQueue()) { + setSendQueue(sender, queue); + + MicrobatchBuffer originalActive = getActiveBuffer(sender); + originalActive.writeByte((byte) 7); + originalActive.incrementRowCount(); + + try { + invokeSealAndSwapBuffer(sender); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("Synthetic enqueue failure")); + } + + // Failed enqueue must not strand the sealed buffer. + Assert.assertSame(originalActive, getActiveBuffer(sender)); + Assert.assertTrue(originalActive.isFilling()); + Assert.assertTrue(originalActive.hasData()); + Assert.assertEquals(1, originalActive.getRowCount()); + + // Retry should be possible on the same sender instance. + invokeSealAndSwapBuffer(sender); + Assert.assertNotSame(originalActive, getActiveBuffer(sender)); + } + }); + } + + @Test + public void testSetGorillaEnabled() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketSender sender = createUnconnectedSender()) { + sender.setGorillaEnabled(false); + Assert.assertFalse(sender.isGorillaEnabled()); + sender.setGorillaEnabled(true); + Assert.assertTrue(sender.isGorillaEnabled()); + } + }); + } + + @Test + public void testStringColumnAfterCloseThrows() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.stringColumn("x", "test"); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + }); + } + + @Test + public void testSymbolAfterCloseThrows() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.symbol("x", "test"); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + }); + } + + @Test + public void testTableBeforeAtNowRequired() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketSender sender = createUnconnectedSender()) { + sender.atNow(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("table()")); + } + }); + } + + @Test + public void testTableBeforeAtRequired() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketSender sender = createUnconnectedSender()) { + sender.at(1000L, ChronoUnit.MICROS); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("table()")); + } + }); + } + + @Test + public void testTableBeforeColumnsRequired() throws Exception { + assertMemoryLeak(() -> { + // Create sender without connecting (we'll catch the error earlier) + try (QwpWebSocketSender sender = createUnconnectedSender()) { + sender.longColumn("x", 1); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("table()")); + } + }); + } + + @Test + public void testTimestampColumnAfterCloseThrows() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.timestampColumn("x", 1000L, ChronoUnit.MICROS); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + }); + } + + @Test + public void testTimestampColumnInstantAfterCloseThrows() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.timestampColumn("x", Instant.now()); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + }); + } + + private static MicrobatchBuffer getActiveBuffer(QwpWebSocketSender sender) throws Exception { + Field field = QwpWebSocketSender.class.getDeclaredField("activeBuffer"); + field.setAccessible(true); + return (MicrobatchBuffer) field.get(sender); + } + + private static void invokeSealAndSwapBuffer(QwpWebSocketSender sender) throws Exception { + Method method = QwpWebSocketSender.class.getDeclaredMethod("sealAndSwapBuffer"); + method.setAccessible(true); + try { + method.invoke(sender); + } catch (InvocationTargetException e) { + Throwable cause = e.getCause(); + if (cause instanceof Exception) { + throw (Exception) cause; + } + if (cause instanceof Error) { + throw (Error) cause; + } + throw new RuntimeException(cause); + } + } + + private static void setSendQueue(QwpWebSocketSender sender, WebSocketSendQueue queue) throws Exception { + Field field = QwpWebSocketSender.class.getDeclaredField("sendQueue"); + field.setAccessible(true); + field.set(sender, queue); + } + + /** + * Creates an async sender without connecting. + */ + private QwpWebSocketSender createUnconnectedAsyncSender() { + return QwpWebSocketSender.createForTesting("localhost", 9000, + 500, 0, 0L, // autoFlushRows, autoFlushBytes, autoFlushIntervalNanos + 8); // inFlightWindowSize + } + + /** + * Creates an async sender with custom flow control settings without connecting. + */ + private QwpWebSocketSender createUnconnectedAsyncSenderWithFlowControl( + int autoFlushRows, int autoFlushBytes, long autoFlushIntervalNanos, + int inFlightWindowSize) { + return QwpWebSocketSender.createForTesting("localhost", 9000, + autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, + inFlightWindowSize); + } + + /** + * Creates a sender without connecting. + * For unit tests that don't need actual connectivity. + */ + private QwpWebSocketSender createUnconnectedSender() { + return QwpWebSocketSender.createForTesting("localhost", 9000, 1); // window=1 for sync + } + + private static class NoOpWebSocketClient extends WebSocketClient { + private NoOpWebSocketClient() { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + } + + @Override + public boolean isConnected() { + return false; + } + + @Override + public void sendBinary(long dataPtr, int length) { + // no-op + } + + @Override + protected void ioWait(int timeout, int op) { + // no-op + } + + @Override + protected void setupIoWait() { + // no-op + } + } + + private static class ThrowingOnceWebSocketSendQueue extends WebSocketSendQueue { + private boolean failOnce = true; + + private ThrowingOnceWebSocketSendQueue() { + super(new NoOpWebSocketClient(), null, 50, 50); + } + + @Override + public boolean enqueue(MicrobatchBuffer buffer) { + if (failOnce) { + failOnce = false; + throw new LineSenderException("Synthetic enqueue failure"); + } + return true; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketSendQueueTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketSendQueueTest.java new file mode 100644 index 0000000..91638d8 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketSendQueueTest.java @@ -0,0 +1,471 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.InFlightWindow; +import io.questdb.client.cutlass.qwp.client.MicrobatchBuffer; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.client.WebSocketSendQueue; +import io.questdb.client.network.PlainSocketFactory; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.*; + +public class WebSocketSendQueueTest { + + @Test + public void testEnqueueTimeoutWhenPendingSlotOccupied() throws Exception { + assertMemoryLeak(() -> { + InFlightWindow window = new InFlightWindow(1, 1_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + MicrobatchBuffer batch0 = sealedBuffer((byte) 1); + MicrobatchBuffer batch1 = sealedBuffer((byte) 2); + WebSocketSendQueue queue = null; + + try { + // Keep window full so I/O thread cannot drain pending slot. + window.addInFlight(0); + queue = new WebSocketSendQueue(client, window, 100, 500); + queue.enqueue(batch0); + + try { + queue.enqueue(batch1); + fail("Expected enqueue timeout"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("Enqueue timeout")); + } + } finally { + window.acknowledgeUpTo(Long.MAX_VALUE); + closeQuietly(queue); + batch0.close(); + batch1.close(); + client.close(); + } + }); + } + + @Test + public void testEnqueueWaitsUntilSlotAvailable() throws Exception { + assertMemoryLeak(() -> { + InFlightWindow window = new InFlightWindow(1, 1_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + MicrobatchBuffer batch0 = sealedBuffer((byte) 1); + MicrobatchBuffer batch1 = sealedBuffer((byte) 2); + WebSocketSendQueue queue = null; + + try { + window.addInFlight(0); + queue = new WebSocketSendQueue(client, window, 2_000, 500); + final WebSocketSendQueue finalQueue = queue; + queue.enqueue(batch0); + + CountDownLatch started = new CountDownLatch(1); + CountDownLatch finished = new CountDownLatch(1); + AtomicReference errorRef = new AtomicReference<>(); + + Thread t = new Thread(() -> { + started.countDown(); + try { + finalQueue.enqueue(batch1); + } catch (Throwable t1) { + errorRef.set(t1); + } finally { + finished.countDown(); + } + }); + t.start(); + + assertTrue(started.await(1, TimeUnit.SECONDS)); + awaitThreadBlocked(t); + assertEquals("Second enqueue should still be waiting", 1, finished.getCount()); + + // Free space so I/O thread can poll pending slot. + window.acknowledgeUpTo(0); + + assertTrue("Second enqueue should complete", finished.await(2, TimeUnit.SECONDS)); + assertNull(errorRef.get()); + } finally { + window.acknowledgeUpTo(Long.MAX_VALUE); + closeQuietly(queue); + batch0.close(); + batch1.close(); + client.close(); + } + }); + } + + @Test + public void testFlushFailsOnInvalidAckPayload() throws Exception { + assertMemoryLeak(() -> { + InFlightWindow window = new InFlightWindow(8, 5_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + WebSocketSendQueue queue = null; + CountDownLatch payloadDelivered = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(false); + + try { + window.addInFlight(0); + client.setTryReceiveBehavior(handler -> { + if (fired.compareAndSet(false, true)) { + emitBinary(handler, new byte[]{1, 2, 3}); + payloadDelivered.countDown(); + return true; + } + return false; + }); + + queue = new WebSocketSendQueue(client, window, 1_000, 500); + assertTrue("Expected invalid payload callback", payloadDelivered.await(2, TimeUnit.SECONDS)); + + try { + queue.flush(); + fail("Expected flush failure on invalid payload"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("Invalid ACK response payload")); + } + } finally { + closeQuietly(queue); + client.close(); + } + }); + } + + @Test + public void testFlushFailsOnReceiveIoError() throws Exception { + assertMemoryLeak(() -> { + InFlightWindow window = new InFlightWindow(8, 5_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + WebSocketSendQueue queue = null; + CountDownLatch receiveAttempted = new CountDownLatch(1); + + try { + window.addInFlight(0); + client.setTryReceiveBehavior(handler -> { + receiveAttempted.countDown(); + throw new RuntimeException("recv-fail"); + }); + + queue = new WebSocketSendQueue(client, window, 1_000, 500); + assertTrue("Expected receive attempt", receiveAttempted.await(2, TimeUnit.SECONDS)); + long deadline = System.currentTimeMillis() + 2_000; + while (queue.getLastError() == null && System.currentTimeMillis() < deadline) { + Thread.sleep(5); + } + assertNotNull("Expected queue error after receive failure", queue.getLastError()); + + try { + queue.flush(); + fail("Expected flush failure after receive error"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("Error receiving response")); + } + } finally { + closeQuietly(queue); + client.close(); + } + }); + } + + @Test + public void testFlushFailsOnSendIoError() throws Exception { + assertMemoryLeak(() -> { + FakeWebSocketClient client = new FakeWebSocketClient(); + MicrobatchBuffer batch = sealedBuffer((byte) 42); + WebSocketSendQueue queue = null; + + try { + client.setSendBehavior((dataPtr, length) -> { + throw new RuntimeException("send-fail"); + }); + queue = new WebSocketSendQueue(client, null, 1_000, 500); + queue.enqueue(batch); + + try { + queue.flush(); + fail("Expected flush failure after send error"); + } catch (LineSenderException e) { + assertTrue( + e.getMessage().contains("Error sending batch") + || e.getMessage().contains("Error in send queue I/O thread") + ); + } + } finally { + closeQuietly(queue); + batch.close(); + client.close(); + } + }); + } + + @Test + public void testFlushFailsWhenServerClosesConnection() throws Exception { + assertMemoryLeak(() -> { + InFlightWindow window = new InFlightWindow(8, 5_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + WebSocketSendQueue queue = null; + CountDownLatch closeDelivered = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(false); + + try { + window.addInFlight(0); + client.setTryReceiveBehavior(handler -> { + if (fired.compareAndSet(false, true)) { + handler.onClose(1006, "boom"); + closeDelivered.countDown(); + return true; + } + return false; + }); + + queue = new WebSocketSendQueue(client, window, 1_000, 500); + assertTrue("Expected close callback", closeDelivered.await(2, TimeUnit.SECONDS)); + + try { + queue.flush(); + fail("Expected flush failure after close"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("closed")); + } + } finally { + closeQuietly(queue); + client.close(); + } + }); + } + + @Test + public void testAwaitPendingAcksKeepsDrainNonBlocking() throws Exception { + assertMemoryLeak(() -> { + InFlightWindow window = new InFlightWindow(8, 5_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + WebSocketSendQueue queue = null; + MicrobatchBuffer batch0 = sealedBuffer((byte) 1); + MicrobatchBuffer batch1 = sealedBuffer((byte) 2); + CountDownLatch secondBatchSent = new CountDownLatch(1); + AtomicBoolean deliverAcks = new AtomicBoolean(false); + AtomicInteger tryReceivePolls = new AtomicInteger(); + AtomicLong highestSent = new AtomicLong(-1); + AtomicReference errorRef = new AtomicReference<>(); + + try { + client.setSendBehavior((dataPtr, length) -> { + long sent = highestSent.incrementAndGet(); + if (sent == 1) { + secondBatchSent.countDown(); + } + }); + client.setReceiveBehavior((handler, timeout) -> { + throw new AssertionError("receiveFrame() must not be used while draining ACKs"); + }); + client.setTryReceiveBehavior(handler -> { + tryReceivePolls.incrementAndGet(); + if (deliverAcks.get()) { + long sent = highestSent.get(); + if (sent >= 0 && window.getInFlightCount() > 0) { + emitAck(handler, sent); + return true; + } + } + return false; + }); + + queue = new WebSocketSendQueue(client, window, 1_000, 500); + queue.enqueue(batch0); + queue.flush(); + + CountDownLatch finished = new CountDownLatch(1); + WebSocketSendQueue finalQueue = queue; + Thread waiter = new Thread(() -> { + try { + finalQueue.awaitPendingAcks(); + } catch (Throwable t) { + errorRef.set(t); + } finally { + finished.countDown(); + } + }); + waiter.start(); + + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (tryReceivePolls.get() == 0 && System.nanoTime() < deadline) { + Thread.onSpinWait(); + } + assertTrue("Expected non-blocking ACK polls while draining", tryReceivePolls.get() > 0); + + queue.enqueue(batch1); + assertTrue("I/O thread should still send new work while ACK drain is active", + secondBatchSent.await(1, TimeUnit.SECONDS)); + + deliverAcks.set(true); + + assertTrue("awaitPendingAcks should complete once ACK arrives", + finished.await(2, TimeUnit.SECONDS)); + assertNull(errorRef.get()); + assertEquals(0, window.getInFlightCount()); + } finally { + closeQuietly(queue); + batch0.close(); + batch1.close(); + client.close(); + } + }); + } + + private static void awaitThreadBlocked(Thread thread) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (System.nanoTime() < deadline) { + Thread.State state = thread.getState(); + if (state == Thread.State.WAITING || state == Thread.State.TIMED_WAITING) { + return; + } + Thread.sleep(1); + } + fail("Thread did not reach blocked state within 5s, state: " + thread.getState()); + } + + private static void closeQuietly(WebSocketSendQueue queue) { + if (queue != null) { + queue.close(); + } + } + + private static void emitBinary(WebSocketFrameHandler handler, byte[] payload) { + long ptr = Unsafe.malloc(payload.length, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < payload.length; i++) { + Unsafe.getUnsafe().putByte(ptr + i, payload[i]); + } + handler.onBinaryMessage(ptr, payload.length); + } finally { + Unsafe.free(ptr, payload.length, MemoryTag.NATIVE_DEFAULT); + } + } + + private static void emitAck(WebSocketFrameHandler handler, long sequence) { + WebSocketResponse response = WebSocketResponse.success(sequence); + int size = response.serializedSize(); + long ptr = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); + try { + response.writeTo(ptr); + handler.onBinaryMessage(ptr, size); + } finally { + Unsafe.free(ptr, size, MemoryTag.NATIVE_DEFAULT); + } + } + + private static MicrobatchBuffer sealedBuffer(byte value) { + MicrobatchBuffer buffer = new MicrobatchBuffer(64); + buffer.writeByte(value); + buffer.incrementRowCount(); + buffer.seal(); + return buffer; + } + + private interface SendBehavior { + void send(long dataPtr, int length); + } + + private interface TryReceiveBehavior { + boolean tryReceive(WebSocketFrameHandler handler); + } + + private interface ReceiveBehavior { + boolean receive(WebSocketFrameHandler handler, int timeout); + } + + private static class FakeWebSocketClient extends WebSocketClient { + private volatile TryReceiveBehavior behavior = handler -> false; + private volatile boolean connected = true; + private volatile ReceiveBehavior receiveBehavior = (handler, timeout) -> false; + private volatile SendBehavior sendBehavior = (dataPtr, length) -> { + }; + + private FakeWebSocketClient() { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + } + + @Override + public void close() { + connected = false; + super.close(); + } + + @Override + public boolean isConnected() { + return connected; + } + + @Override + public void sendBinary(long dataPtr, int length) { + sendBehavior.send(dataPtr, length); + } + + public void setSendBehavior(SendBehavior sendBehavior) { + this.sendBehavior = sendBehavior; + } + + public void setTryReceiveBehavior(TryReceiveBehavior behavior) { + this.behavior = behavior; + } + + public void setReceiveBehavior(ReceiveBehavior receiveBehavior) { + this.receiveBehavior = receiveBehavior; + } + + @Override + public boolean receiveFrame(WebSocketFrameHandler handler, int timeout) { + return receiveBehavior.receive(handler, timeout); + } + + @Override + public boolean tryReceiveFrame(WebSocketFrameHandler handler) { + return behavior.tryReceive(handler); + } + + @Override + protected void ioWait(int timeout, int op) { + // no-op + } + + @Override + protected void setupIoWait() { + // no-op + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java new file mode 100644 index 0000000..8fae113 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java @@ -0,0 +1,405 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.qwp.protocol.OffHeapAppendMemory; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class OffHeapAppendMemoryTest { + + @Test + public void testCloseFreesMemory() throws Exception { + assertMemoryLeak(() -> { + long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + OffHeapAppendMemory mem = new OffHeapAppendMemory(1024); + long during = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + assertTrue(during > before); + + mem.close(); + long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + assertEquals(before, after); + }); + } + + @Test + public void testDoubleCloseIsSafe() throws Exception { + assertMemoryLeak(() -> { + OffHeapAppendMemory mem = new OffHeapAppendMemory(); + mem.putInt(42); + mem.close(); + mem.close(); // should not throw + }); + } + + @Test + public void testGrowth() throws Exception { + assertMemoryLeak(() -> { + long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + try (OffHeapAppendMemory mem = new OffHeapAppendMemory(8)) { + // Write more data than initial capacity to force growth + for (int i = 0; i < 100; i++) { + mem.putLong(i); + } + + assertEquals(800, mem.getAppendOffset()); + for (int i = 0; i < 100; i++) { + assertEquals(i, Unsafe.getUnsafe().getLong(mem.addressOf((long) i * 8))); + } + } + long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + assertEquals(before, after); + }); + } + + @Test + public void testJumpTo() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putLong(100); + mem.putLong(200); + mem.putLong(300); + assertEquals(24, mem.getAppendOffset()); + + // Jump back to offset 8 (after first long) + mem.jumpTo(8); + assertEquals(8, mem.getAppendOffset()); + + // Write new value at offset 8 + mem.putLong(999); + assertEquals(16, mem.getAppendOffset()); + assertEquals(100, Unsafe.getUnsafe().getLong(mem.addressOf(0))); + assertEquals(999, Unsafe.getUnsafe().getLong(mem.addressOf(8))); + } + }); + } + + @Test + public void testLargeGrowth() throws Exception { + assertMemoryLeak(() -> { + long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + try (OffHeapAppendMemory mem = new OffHeapAppendMemory(8)) { + // Write 10000 doubles to stress growth + for (int i = 0; i < 10_000; i++) { + mem.putDouble(i * 1.1); + } + assertEquals(80_000, mem.getAppendOffset()); + + // Verify first and last values + assertEquals(0.0, Unsafe.getUnsafe().getDouble(mem.addressOf(0)), 0.0); + assertEquals(9999 * 1.1, Unsafe.getUnsafe().getDouble(mem.addressOf(79_992)), 0.001); + } + long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + assertEquals(before, after); + }); + } + + @Test + public void testMixedTypes() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putByte((byte) 1); + mem.putShort((short) 2); + mem.putInt(3); + mem.putLong(4L); + mem.putFloat(5.0f); + mem.putDouble(6.0); + + long addr = mem.pageAddress(); + assertEquals(1, Unsafe.getUnsafe().getByte(addr)); + assertEquals(2, Unsafe.getUnsafe().getShort(addr + 1)); + assertEquals(3, Unsafe.getUnsafe().getInt(addr + 3)); + assertEquals(4L, Unsafe.getUnsafe().getLong(addr + 7)); + assertEquals(5.0f, Unsafe.getUnsafe().getFloat(addr + 15), 0.0f); + assertEquals(6.0, Unsafe.getUnsafe().getDouble(addr + 19), 0.0); + assertEquals(27, mem.getAppendOffset()); + } + }); + } + + @Test + public void testPageAddress() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + assertTrue(mem.pageAddress() != 0); + assertEquals(mem.pageAddress(), mem.addressOf(0)); + mem.putLong(42); + assertEquals(mem.pageAddress() + 8, mem.addressOf(8)); + } + }); + } + + @Test + public void testPutAndReadByte() throws Exception { + assertMemoryLeak(() -> { + long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putByte((byte) 42); + mem.putByte((byte) -1); + mem.putByte((byte) 0); + + assertEquals(3, mem.getAppendOffset()); + assertEquals(42, Unsafe.getUnsafe().getByte(mem.addressOf(0))); + assertEquals(-1, Unsafe.getUnsafe().getByte(mem.addressOf(1))); + assertEquals(0, Unsafe.getUnsafe().getByte(mem.addressOf(2))); + } + long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + assertEquals(before, after); + }); + } + + @Test + public void testPutAndReadDouble() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putDouble(2.718281828); + mem.putDouble(Double.NaN); + + assertEquals(16, mem.getAppendOffset()); + assertEquals(2.718281828, Unsafe.getUnsafe().getDouble(mem.addressOf(0)), 0.0); + assertTrue(Double.isNaN(Unsafe.getUnsafe().getDouble(mem.addressOf(8)))); + } + }); + } + + @Test + public void testPutAndReadFloat() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putFloat(3.14f); + mem.putFloat(Float.NaN); + + assertEquals(8, mem.getAppendOffset()); + assertEquals(3.14f, Unsafe.getUnsafe().getFloat(mem.addressOf(0)), 0.0f); + assertTrue(Float.isNaN(Unsafe.getUnsafe().getFloat(mem.addressOf(4)))); + } + }); + } + + @Test + public void testPutAndReadInt() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putInt(100_000); + mem.putInt(Integer.MIN_VALUE); + + assertEquals(8, mem.getAppendOffset()); + assertEquals(100_000, Unsafe.getUnsafe().getInt(mem.addressOf(0))); + assertEquals(Integer.MIN_VALUE, Unsafe.getUnsafe().getInt(mem.addressOf(4))); + } + }); + } + + @Test + public void testPutAndReadLong() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putLong(1_000_000_000_000L); + mem.putLong(Long.MIN_VALUE); + + assertEquals(16, mem.getAppendOffset()); + assertEquals(1_000_000_000_000L, Unsafe.getUnsafe().getLong(mem.addressOf(0))); + assertEquals(Long.MIN_VALUE, Unsafe.getUnsafe().getLong(mem.addressOf(8))); + } + }); + } + + @Test + public void testPutAndReadShort() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putShort((short) 12_345); + mem.putShort(Short.MIN_VALUE); + mem.putShort(Short.MAX_VALUE); + + assertEquals(6, mem.getAppendOffset()); + assertEquals(12_345, Unsafe.getUnsafe().getShort(mem.addressOf(0))); + assertEquals(Short.MIN_VALUE, Unsafe.getUnsafe().getShort(mem.addressOf(2))); + assertEquals(Short.MAX_VALUE, Unsafe.getUnsafe().getShort(mem.addressOf(4))); + } + }); + } + + @Test + public void testPutBoolean() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putBoolean(true); + mem.putBoolean(false); + mem.putBoolean(true); + + assertEquals(3, mem.getAppendOffset()); + assertEquals(1, Unsafe.getUnsafe().getByte(mem.addressOf(0))); + assertEquals(0, Unsafe.getUnsafe().getByte(mem.addressOf(1))); + assertEquals(1, Unsafe.getUnsafe().getByte(mem.addressOf(2))); + } + }); + } + + @Test + public void testPutUtf8Ascii() throws Exception { + assertMemoryLeak(() -> { + long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putUtf8("hello"); + assertEquals(5, mem.getAppendOffset()); + assertEquals('h', Unsafe.getUnsafe().getByte(mem.addressOf(0))); + assertEquals('e', Unsafe.getUnsafe().getByte(mem.addressOf(1))); + assertEquals('l', Unsafe.getUnsafe().getByte(mem.addressOf(2))); + assertEquals('l', Unsafe.getUnsafe().getByte(mem.addressOf(3))); + assertEquals('o', Unsafe.getUnsafe().getByte(mem.addressOf(4))); + } + long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + assertEquals(before, after); + }); + } + + @Test + public void testPutUtf8Empty() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putUtf8(""); + assertEquals(0, mem.getAppendOffset()); + } + }); + } + + @Test + public void testPutUtf8Mixed() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + // Mix: ASCII "A" (1 byte) + e-acute (2 bytes) + CJK (3 bytes) + emoji (4 bytes) = 10 bytes + mem.putUtf8("A\u00E9\u4E16\uD83D\uDE00"); + assertEquals(10, mem.getAppendOffset()); + } + }); + } + + @Test + public void testPutUtf8MultiByte() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + // 2-byte: U+00E9 (e-acute) = C3 A9 + mem.putUtf8("\u00E9"); + assertEquals(2, mem.getAppendOffset()); + assertEquals((byte) 0xC3, Unsafe.getUnsafe().getByte(mem.addressOf(0))); + assertEquals((byte) 0xA9, Unsafe.getUnsafe().getByte(mem.addressOf(1))); + } + }); + } + + @Test + public void testPutUtf8Null() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putUtf8(null); + assertEquals(0, mem.getAppendOffset()); + } + }); + } + + @Test + public void testPutUtf8InvalidSurrogatePair() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + // High surrogate \uD800 followed by non-low-surrogate 'X'. + // Should produce '?' for the lone high surrogate, then 'X'. + mem.putUtf8("\uD800X"); + assertEquals(2, mem.getAppendOffset()); + assertEquals((byte) '?', Unsafe.getUnsafe().getByte(mem.addressOf(0))); + assertEquals((byte) 'X', Unsafe.getUnsafe().getByte(mem.addressOf(1))); + } + }); + } + + @Test + public void testPutUtf8SurrogatePairs() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + // U+1F600 (grinning face) = F0 9F 98 80 + mem.putUtf8("\uD83D\uDE00"); + assertEquals(4, mem.getAppendOffset()); + assertEquals((byte) 0xF0, Unsafe.getUnsafe().getByte(mem.addressOf(0))); + assertEquals((byte) 0x9F, Unsafe.getUnsafe().getByte(mem.addressOf(1))); + assertEquals((byte) 0x98, Unsafe.getUnsafe().getByte(mem.addressOf(2))); + assertEquals((byte) 0x80, Unsafe.getUnsafe().getByte(mem.addressOf(3))); + } + }); + } + + @Test + public void testPutUtf8ThreeByte() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + // 3-byte: U+4E16 (CJK character) = E4 B8 96 + mem.putUtf8("\u4E16"); + assertEquals(3, mem.getAppendOffset()); + assertEquals((byte) 0xE4, Unsafe.getUnsafe().getByte(mem.addressOf(0))); + assertEquals((byte) 0xB8, Unsafe.getUnsafe().getByte(mem.addressOf(1))); + assertEquals((byte) 0x96, Unsafe.getUnsafe().getByte(mem.addressOf(2))); + } + }); + } + + @Test + public void testSkip() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putInt(1); + mem.skip(8); + mem.putInt(2); + + assertEquals(16, mem.getAppendOffset()); + assertEquals(1, Unsafe.getUnsafe().getInt(mem.addressOf(0))); + assertEquals(2, Unsafe.getUnsafe().getInt(mem.addressOf(12))); + } + }); + } + + @Test + public void testTruncate() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putInt(1); + mem.putInt(2); + mem.putInt(3); + assertEquals(12, mem.getAppendOffset()); + + mem.truncate(); + assertEquals(0, mem.getAppendOffset()); + + // Can write again after truncate + mem.putInt(42); + assertEquals(4, mem.getAppendOffset()); + assertEquals(42, Unsafe.getUnsafe().getInt(mem.addressOf(0))); + } + }); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpBitWriterTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpBitWriterTest.java new file mode 100644 index 0000000..2e539dc --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpBitWriterTest.java @@ -0,0 +1,206 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.protocol.QwpBitWriter; +import io.questdb.client.cutlass.qwp.protocol.QwpGorillaEncoder; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class QwpBitWriterTest { + + @Test + public void testFlushThrowsOnOverflow() throws Exception { + assertMemoryLeak(() -> { + long ptr = Unsafe.malloc(1, MemoryTag.NATIVE_ILP_RSS); + try { + QwpBitWriter writer = new QwpBitWriter(); + writer.reset(ptr, 1); + // Write 8 bits to fill the single byte + writer.writeBits(0xFF, 8); + // Write a few more bits that sit in the bit buffer + writer.writeBits(0x3, 4); + // Flush should throw because there's no room for the partial byte + try { + writer.flush(); + fail("expected LineSenderException on buffer overflow during flush"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("buffer overflow")); + } + } finally { + Unsafe.free(ptr, 1, MemoryTag.NATIVE_ILP_RSS); + } + }); + } + + @Test + public void testGorillaEncoderThrowsOnInsufficientCapacityForFirstTimestamp() throws Exception { + assertMemoryLeak(() -> { + // Source: 1 timestamp (8 bytes), dest: only 4 bytes + long src = Unsafe.malloc(8, MemoryTag.NATIVE_ILP_RSS); + long dst = Unsafe.malloc(4, MemoryTag.NATIVE_ILP_RSS); + try { + Unsafe.getUnsafe().putLong(src, 1_000_000L); + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + try { + encoder.encodeTimestamps(dst, 4, src, 1); + fail("expected LineSenderException on buffer overflow"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("buffer overflow")); + } + } finally { + Unsafe.free(src, 8, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, 4, MemoryTag.NATIVE_ILP_RSS); + } + }); + } + + @Test + public void testGorillaEncoderThrowsOnInsufficientCapacityForSecondTimestamp() throws Exception { + assertMemoryLeak(() -> { + // Source: 2 timestamps (16 bytes), dest: only 12 bytes (enough for first, not second) + long src = Unsafe.malloc(16, MemoryTag.NATIVE_ILP_RSS); + long dst = Unsafe.malloc(12, MemoryTag.NATIVE_ILP_RSS); + try { + Unsafe.getUnsafe().putLong(src, 1_000_000L); + Unsafe.getUnsafe().putLong(src + 8, 2_000_000L); + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + try { + encoder.encodeTimestamps(dst, 12, src, 2); + fail("expected LineSenderException on buffer overflow"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("buffer overflow")); + } + } finally { + Unsafe.free(src, 16, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, 12, MemoryTag.NATIVE_ILP_RSS); + } + }); + } + + @Test + public void testWriteBitsThrowsOnOverflow() throws Exception { + assertMemoryLeak(() -> { + long ptr = Unsafe.malloc(4, MemoryTag.NATIVE_ILP_RSS); + try { + QwpBitWriter writer = new QwpBitWriter(); + writer.reset(ptr, 4); + // Fill the buffer (32 bits = 4 bytes) + writer.writeBits(0xFFFF_FFFFL, 32); + // Next write should throw — buffer is full + try { + writer.writeBits(1, 8); + fail("expected LineSenderException on buffer overflow"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("buffer overflow")); + } + } finally { + Unsafe.free(ptr, 4, MemoryTag.NATIVE_ILP_RSS); + } + }); + } + + @Test + public void testWriteBitsWithinCapacitySucceeds() throws Exception { + assertMemoryLeak(() -> { + long ptr = Unsafe.malloc(8, MemoryTag.NATIVE_ILP_RSS); + try { + QwpBitWriter writer = new QwpBitWriter(); + writer.reset(ptr, 8); + writer.writeBits(0xDEAD_BEEF_CAFE_BABEL, 64); + writer.flush(); + assertEquals(8, writer.getPosition() - ptr); + assertEquals(0xDEAD_BEEF_CAFE_BABEL, Unsafe.getUnsafe().getLong(ptr)); + } finally { + Unsafe.free(ptr, 8, MemoryTag.NATIVE_ILP_RSS); + } + }); + } + + @Test + public void testWriteByteThrowsOnOverflow() throws Exception { + assertMemoryLeak(() -> { + long ptr = Unsafe.malloc(1, MemoryTag.NATIVE_ILP_RSS); + try { + QwpBitWriter writer = new QwpBitWriter(); + writer.reset(ptr, 1); + writer.writeByte(0x42); + try { + writer.writeByte(0x43); + fail("expected LineSenderException on buffer overflow"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("buffer overflow")); + } + } finally { + Unsafe.free(ptr, 1, MemoryTag.NATIVE_ILP_RSS); + } + }); + } + + @Test + public void testWriteIntThrowsOnOverflow() throws Exception { + assertMemoryLeak(() -> { + long ptr = Unsafe.malloc(4, MemoryTag.NATIVE_ILP_RSS); + try { + QwpBitWriter writer = new QwpBitWriter(); + writer.reset(ptr, 4); + writer.writeInt(42); + try { + writer.writeInt(99); + fail("expected LineSenderException on buffer overflow"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("buffer overflow")); + } + } finally { + Unsafe.free(ptr, 4, MemoryTag.NATIVE_ILP_RSS); + } + }); + } + + @Test + public void testWriteLongThrowsOnOverflow() throws Exception { + assertMemoryLeak(() -> { + long ptr = Unsafe.malloc(8, MemoryTag.NATIVE_ILP_RSS); + try { + QwpBitWriter writer = new QwpBitWriter(); + writer.reset(ptr, 8); + writer.writeLong(42L); + try { + writer.writeLong(99L); + fail("expected LineSenderException on buffer overflow"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("buffer overflow")); + } + } finally { + Unsafe.free(ptr, 8, MemoryTag.NATIVE_ILP_RSS); + } + }); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpColumnDefTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpColumnDefTest.java new file mode 100644 index 0000000..b69c646 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpColumnDefTest.java @@ -0,0 +1,95 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.qwp.protocol.QwpColumnDef; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class QwpColumnDefTest { + + @Test + public void testValidateAcceptsAllValidTypes() { + byte[] validTypes = { + QwpConstants.TYPE_BOOLEAN, + QwpConstants.TYPE_BYTE, + QwpConstants.TYPE_SHORT, + QwpConstants.TYPE_INT, + QwpConstants.TYPE_LONG, + QwpConstants.TYPE_FLOAT, + QwpConstants.TYPE_DOUBLE, + QwpConstants.TYPE_STRING, + QwpConstants.TYPE_SYMBOL, + QwpConstants.TYPE_TIMESTAMP, + QwpConstants.TYPE_DATE, + QwpConstants.TYPE_UUID, + QwpConstants.TYPE_LONG256, + QwpConstants.TYPE_GEOHASH, + QwpConstants.TYPE_VARCHAR, + QwpConstants.TYPE_TIMESTAMP_NANOS, + QwpConstants.TYPE_DOUBLE_ARRAY, + QwpConstants.TYPE_LONG_ARRAY, + QwpConstants.TYPE_DECIMAL64, + QwpConstants.TYPE_DECIMAL128, + QwpConstants.TYPE_DECIMAL256, + QwpConstants.TYPE_CHAR, + }; + for (byte type : validTypes) { + QwpColumnDef col = new QwpColumnDef("col", type); + col.validate(); // must not throw + } + } + + @Test + public void testValidateCharType() { + // TYPE_CHAR (0x16) must pass validation + QwpColumnDef col = new QwpColumnDef("ch", QwpConstants.TYPE_CHAR); + col.validate(); + assertEquals("CHAR", col.getTypeName()); + assertEquals(QwpConstants.TYPE_CHAR, col.getTypeCode()); + } + + @Test(expected = IllegalArgumentException.class) + public void testValidateRejectsHighBit() { + // The high bit is not a valid part of the type code + byte badType = (byte) (QwpConstants.TYPE_CHAR | 0x80); + QwpColumnDef col = new QwpColumnDef("ch", badType); + col.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void testValidateRejectsInvalidType() { + QwpColumnDef col = new QwpColumnDef("bad", (byte) 0x17); + col.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void testValidateRejectsZeroType() { + QwpColumnDef col = new QwpColumnDef("bad", (byte) 0x00); + col.validate(); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java new file mode 100644 index 0000000..9bb14c0 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java @@ -0,0 +1,219 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import org.junit.Assert; +import org.junit.Test; + +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; + +public class QwpConstantsTest { + + @Test + public void testDefaultLimits() { + Assert.assertEquals(16 * 1024 * 1024, DEFAULT_MAX_BATCH_SIZE); + Assert.assertEquals(256, DEFAULT_MAX_TABLES_PER_BATCH); + Assert.assertEquals(1_000_000, DEFAULT_MAX_ROWS_PER_TABLE); + Assert.assertEquals(2048, MAX_COLUMNS_PER_TABLE); + Assert.assertEquals(64 * 1024, DEFAULT_INITIAL_RECV_BUFFER_SIZE); + Assert.assertEquals(4, DEFAULT_MAX_IN_FLIGHT_BATCHES); + } + + @Test + public void testFlagBitPositions() { + // Verify flag bits are at correct positions + Assert.assertEquals(0x01, FLAG_LZ4); + Assert.assertEquals(0x02, FLAG_ZSTD); + Assert.assertEquals(0x04, FLAG_GORILLA); + Assert.assertEquals(0x03, FLAG_COMPRESSION_MASK); + Assert.assertEquals(0x08, FLAG_DELTA_SYMBOL_DICT); + } + + @Test + public void testGetFixedTypeSize() { + Assert.assertEquals(0, QwpConstants.getFixedTypeSize(TYPE_BOOLEAN)); // Bit-packed + Assert.assertEquals(1, QwpConstants.getFixedTypeSize(TYPE_BYTE)); + Assert.assertEquals(2, QwpConstants.getFixedTypeSize(TYPE_SHORT)); + Assert.assertEquals(2, QwpConstants.getFixedTypeSize(TYPE_CHAR)); + Assert.assertEquals(4, QwpConstants.getFixedTypeSize(TYPE_INT)); + Assert.assertEquals(8, QwpConstants.getFixedTypeSize(TYPE_LONG)); + Assert.assertEquals(4, QwpConstants.getFixedTypeSize(TYPE_FLOAT)); + Assert.assertEquals(8, QwpConstants.getFixedTypeSize(TYPE_DOUBLE)); + Assert.assertEquals(8, QwpConstants.getFixedTypeSize(TYPE_TIMESTAMP)); + Assert.assertEquals(8, QwpConstants.getFixedTypeSize(TYPE_TIMESTAMP_NANOS)); + Assert.assertEquals(8, QwpConstants.getFixedTypeSize(TYPE_DATE)); + Assert.assertEquals(16, QwpConstants.getFixedTypeSize(TYPE_UUID)); + Assert.assertEquals(32, QwpConstants.getFixedTypeSize(TYPE_LONG256)); + Assert.assertEquals(8, QwpConstants.getFixedTypeSize(TYPE_DECIMAL64)); + Assert.assertEquals(16, QwpConstants.getFixedTypeSize(TYPE_DECIMAL128)); + Assert.assertEquals(32, QwpConstants.getFixedTypeSize(TYPE_DECIMAL256)); + + Assert.assertEquals(-1, QwpConstants.getFixedTypeSize(TYPE_STRING)); + Assert.assertEquals(-1, QwpConstants.getFixedTypeSize(TYPE_SYMBOL)); + Assert.assertEquals(-1, QwpConstants.getFixedTypeSize(TYPE_DOUBLE_ARRAY)); + Assert.assertEquals(-1, QwpConstants.getFixedTypeSize(TYPE_LONG_ARRAY)); + } + + @Test + public void testGetTypeName() { + Assert.assertEquals("BOOLEAN", QwpConstants.getTypeName(TYPE_BOOLEAN)); + Assert.assertEquals("INT", QwpConstants.getTypeName(TYPE_INT)); + Assert.assertEquals("STRING", QwpConstants.getTypeName(TYPE_STRING)); + Assert.assertEquals("TIMESTAMP", QwpConstants.getTypeName(TYPE_TIMESTAMP)); + Assert.assertEquals("TIMESTAMP_NANOS", QwpConstants.getTypeName(TYPE_TIMESTAMP_NANOS)); + Assert.assertEquals("DOUBLE_ARRAY", QwpConstants.getTypeName(TYPE_DOUBLE_ARRAY)); + Assert.assertEquals("LONG_ARRAY", QwpConstants.getTypeName(TYPE_LONG_ARRAY)); + Assert.assertEquals("DECIMAL64", QwpConstants.getTypeName(TYPE_DECIMAL64)); + Assert.assertEquals("DECIMAL128", QwpConstants.getTypeName(TYPE_DECIMAL128)); + Assert.assertEquals("DECIMAL256", QwpConstants.getTypeName(TYPE_DECIMAL256)); + Assert.assertEquals("CHAR", QwpConstants.getTypeName(TYPE_CHAR)); + + // Type codes with high bit set are unknown — the high bit is not used + byte badInt = (byte) (TYPE_INT | 0x80); + Assert.assertTrue(QwpConstants.getTypeName(badInt).startsWith("UNKNOWN")); + } + + @Test + public void testHeaderSize() { + Assert.assertEquals(12, HEADER_SIZE); + } + + @Test + public void testIsFixedWidthType() { + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_BOOLEAN)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_BYTE)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_SHORT)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_CHAR)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_INT)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_LONG)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_FLOAT)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_DOUBLE)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_TIMESTAMP)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_TIMESTAMP_NANOS)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_DATE)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_UUID)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_LONG256)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_DECIMAL64)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_DECIMAL128)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_DECIMAL256)); + + Assert.assertFalse(QwpConstants.isFixedWidthType(TYPE_STRING)); + Assert.assertFalse(QwpConstants.isFixedWidthType(TYPE_SYMBOL)); + Assert.assertFalse(QwpConstants.isFixedWidthType(TYPE_GEOHASH)); + Assert.assertFalse(QwpConstants.isFixedWidthType(TYPE_VARCHAR)); + Assert.assertFalse(QwpConstants.isFixedWidthType(TYPE_DOUBLE_ARRAY)); + Assert.assertFalse(QwpConstants.isFixedWidthType(TYPE_LONG_ARRAY)); + } + + @Test + public void testMagicBytesCapabilityRequest() { + // "ILP?" in ASCII + byte[] expected = new byte[]{'I', 'L', 'P', '?'}; + Assert.assertEquals((byte) (MAGIC_CAPABILITY_REQUEST & 0xFF), expected[0]); + Assert.assertEquals((byte) ((MAGIC_CAPABILITY_REQUEST >> 8) & 0xFF), expected[1]); + Assert.assertEquals((byte) ((MAGIC_CAPABILITY_REQUEST >> 16) & 0xFF), expected[2]); + Assert.assertEquals((byte) ((MAGIC_CAPABILITY_REQUEST >> 24) & 0xFF), expected[3]); + } + + @Test + public void testMagicBytesCapabilityResponse() { + // "ILP!" in ASCII + byte[] expected = new byte[]{'I', 'L', 'P', '!'}; + Assert.assertEquals((byte) (MAGIC_CAPABILITY_RESPONSE & 0xFF), expected[0]); + Assert.assertEquals((byte) ((MAGIC_CAPABILITY_RESPONSE >> 8) & 0xFF), expected[1]); + Assert.assertEquals((byte) ((MAGIC_CAPABILITY_RESPONSE >> 16) & 0xFF), expected[2]); + Assert.assertEquals((byte) ((MAGIC_CAPABILITY_RESPONSE >> 24) & 0xFF), expected[3]); + } + + @Test + public void testMagicBytesFallback() { + // "ILP0" in ASCII + byte[] expected = new byte[]{'I', 'L', 'P', '0'}; + Assert.assertEquals((byte) (MAGIC_FALLBACK & 0xFF), expected[0]); + Assert.assertEquals((byte) ((MAGIC_FALLBACK >> 8) & 0xFF), expected[1]); + Assert.assertEquals((byte) ((MAGIC_FALLBACK >> 16) & 0xFF), expected[2]); + Assert.assertEquals((byte) ((MAGIC_FALLBACK >> 24) & 0xFF), expected[3]); + } + + @Test + public void testMagicBytesValue() { + // "QWP1" in ASCII: Q=0x51, W=0x57, P=0x50, 1=0x31 + // Little-endian: 0x31505751 + Assert.assertEquals(0x31505751, MAGIC_MESSAGE); + + // Verify ASCII encoding + byte[] expected = new byte[]{'Q', 'W', 'P', '1'}; + Assert.assertEquals((byte) (MAGIC_MESSAGE & 0xFF), expected[0]); + Assert.assertEquals((byte) ((MAGIC_MESSAGE >> 8) & 0xFF), expected[1]); + Assert.assertEquals((byte) ((MAGIC_MESSAGE >> 16) & 0xFF), expected[2]); + Assert.assertEquals((byte) ((MAGIC_MESSAGE >> 24) & 0xFF), expected[3]); + } + + @Test + public void testSchemaModes() { + Assert.assertEquals(0x00, SCHEMA_MODE_FULL); + Assert.assertEquals(0x01, SCHEMA_MODE_REFERENCE); + } + + @Test + public void testStatusCodes() { + Assert.assertEquals(0x00, STATUS_OK); + Assert.assertEquals(0x01, STATUS_PARTIAL); + Assert.assertEquals(0x02, STATUS_SCHEMA_REQUIRED); + Assert.assertEquals(0x03, STATUS_SCHEMA_MISMATCH); + Assert.assertEquals(0x04, STATUS_TABLE_NOT_FOUND); + Assert.assertEquals(0x05, STATUS_PARSE_ERROR); + Assert.assertEquals(0x06, STATUS_INTERNAL_ERROR); + Assert.assertEquals(0x07, STATUS_OVERLOADED); + } + + @Test + public void testTypeCodes() { + // Verify type codes match specification + Assert.assertEquals(0x01, TYPE_BOOLEAN); + Assert.assertEquals(0x02, TYPE_BYTE); + Assert.assertEquals(0x03, TYPE_SHORT); + Assert.assertEquals(0x04, TYPE_INT); + Assert.assertEquals(0x05, TYPE_LONG); + Assert.assertEquals(0x06, TYPE_FLOAT); + Assert.assertEquals(0x07, TYPE_DOUBLE); + Assert.assertEquals(0x08, TYPE_STRING); + Assert.assertEquals(0x09, TYPE_SYMBOL); + Assert.assertEquals(0x0A, TYPE_TIMESTAMP); + Assert.assertEquals(0x0B, TYPE_DATE); + Assert.assertEquals(0x0C, TYPE_UUID); + Assert.assertEquals(0x0D, TYPE_LONG256); + Assert.assertEquals(0x0E, TYPE_GEOHASH); + Assert.assertEquals(0x0F, TYPE_VARCHAR); + Assert.assertEquals(0x10, TYPE_TIMESTAMP_NANOS); + Assert.assertEquals(0x11, TYPE_DOUBLE_ARRAY); + Assert.assertEquals(0x12, TYPE_LONG_ARRAY); + Assert.assertEquals(0x13, TYPE_DECIMAL64); + Assert.assertEquals(0x14, TYPE_DECIMAL128); + Assert.assertEquals(0x15, TYPE_DECIMAL256); + Assert.assertEquals(0x16, TYPE_CHAR); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashSurrogateTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashSurrogateTest.java new file mode 100644 index 0000000..8fff334 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashSurrogateTest.java @@ -0,0 +1,122 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaHash; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.ObjList; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class QwpSchemaHashSurrogateTest { + + private static final byte TYPE_LONG = 0x05; + + @Test + public void testComputeSchemaHashInvalidSurrogatePair() { + byte[] types = {TYPE_LONG}; + + // "\uD800X" has a high surrogate followed by non-low-surrogate 'X'. + // With the fix, the high surrogate becomes '?' and 'X' is preserved, + // so the hash should equal the hash of "?X". + long hashInvalid = QwpSchemaHash.computeSchemaHash( + new String[]{"\uD800X"}, types + ); + long hashExpected = QwpSchemaHash.computeSchemaHash( + new String[]{"?X"}, types + ); + assertEquals(hashExpected, hashInvalid); + } + + @Test + public void testComputeSchemaHashLoneHighSurrogateAtEnd() { + byte[] types = {TYPE_LONG}; + + // "\uD800" is a lone high surrogate at end of string. + // Must hash as '?' to match OffHeapAppendMemory.putUtf8(). + long hashInvalid = QwpSchemaHash.computeSchemaHash( + new String[]{"col\uD800"}, types + ); + long hashExpected = QwpSchemaHash.computeSchemaHash( + new String[]{"col?"}, types + ); + assertEquals(hashExpected, hashInvalid); + } + + @Test + public void testComputeSchemaHashLoneLowSurrogate() { + byte[] types = {TYPE_LONG}; + + // "\uDC00" is a lone low surrogate (not preceded by a high surrogate). + // Must hash as '?' to match OffHeapAppendMemory.putUtf8(). + long hashInvalid = QwpSchemaHash.computeSchemaHash( + new String[]{"col\uDC00"}, types + ); + long hashExpected = QwpSchemaHash.computeSchemaHash( + new String[]{"col?"}, types + ); + assertEquals(hashExpected, hashInvalid); + } + + @Test + public void testComputeSchemaHashDirectInvalidSurrogatePair() { + ObjList invalidCols = new ObjList<>(); + invalidCols.add(new QwpTableBuffer.ColumnBuffer("\uD800X", TYPE_LONG, false)); + + ObjList expectedCols = new ObjList<>(); + expectedCols.add(new QwpTableBuffer.ColumnBuffer("?X", TYPE_LONG, false)); + + long hashInvalid = QwpSchemaHash.computeSchemaHashDirect(invalidCols); + long hashExpected = QwpSchemaHash.computeSchemaHashDirect(expectedCols); + assertEquals(hashExpected, hashInvalid); + } + + @Test + public void testComputeSchemaHashDirectLoneHighSurrogateAtEnd() { + ObjList invalidCols = new ObjList<>(); + invalidCols.add(new QwpTableBuffer.ColumnBuffer("col\uD800", TYPE_LONG, false)); + + ObjList expectedCols = new ObjList<>(); + expectedCols.add(new QwpTableBuffer.ColumnBuffer("col?", TYPE_LONG, false)); + + long hashInvalid = QwpSchemaHash.computeSchemaHashDirect(invalidCols); + long hashExpected = QwpSchemaHash.computeSchemaHashDirect(expectedCols); + assertEquals(hashExpected, hashInvalid); + } + + @Test + public void testComputeSchemaHashDirectLoneLowSurrogate() { + ObjList invalidCols = new ObjList<>(); + invalidCols.add(new QwpTableBuffer.ColumnBuffer("col\uDC00", TYPE_LONG, false)); + + ObjList expectedCols = new ObjList<>(); + expectedCols.add(new QwpTableBuffer.ColumnBuffer("col?", TYPE_LONG, false)); + + long hashInvalid = QwpSchemaHash.computeSchemaHashDirect(invalidCols); + long hashExpected = QwpSchemaHash.computeSchemaHashDirect(expectedCols); + assertEquals(hashExpected, hashInvalid); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashTest.java new file mode 100644 index 0000000..84b72fe --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashTest.java @@ -0,0 +1,329 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaHash; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; + +public class QwpSchemaHashTest { + + private static final byte TYPE_LONG = 0x05; + + @Test + public void testColumnOrderMatters() { + // Order 1 + String[] names1 = {"price", "symbol"}; + byte[] types1 = {0x07, 0x09}; + + // Order 2 (different order) + String[] names2 = {"symbol", "price"}; + byte[] types2 = {0x09, 0x07}; + + long hash1 = QwpSchemaHash.computeSchemaHash(names1, types1); + long hash2 = QwpSchemaHash.computeSchemaHash(names2, types2); + + Assert.assertNotEquals("Column order should affect hash", hash1, hash2); + } + + @Test + public void testDeterministic() { + String[] names = {"col1", "col2", "col3"}; + byte[] types = {0x01, 0x02, 0x03}; + + long hash1 = QwpSchemaHash.computeSchemaHash(names, types); + long hash2 = QwpSchemaHash.computeSchemaHash(names, types); + long hash3 = QwpSchemaHash.computeSchemaHash(names, types); + + Assert.assertEquals("Hash should be deterministic", hash1, hash2); + Assert.assertEquals("Hash should be deterministic", hash2, hash3); + } + + @Test + public void testEmptySchema() { + String[] names = {}; + byte[] types = {}; + long hash = QwpSchemaHash.computeSchemaHash(names, types); + // Empty input should produce the same hash consistently + Assert.assertEquals(hash, QwpSchemaHash.computeSchemaHash(names, types)); + } + + @Test + public void testHasherReset() { + QwpSchemaHash.Hasher hasher = new QwpSchemaHash.Hasher(); + + byte[] data1 = "first".getBytes(StandardCharsets.UTF_8); + byte[] data2 = "second".getBytes(StandardCharsets.UTF_8); + + // Hash first data + hasher.reset(0); + hasher.update(data1); + long hash1 = hasher.getValue(); + + // Reset and hash second data + hasher.reset(0); + hasher.update(data2); + long hash2 = hasher.getValue(); + + // Should be different + Assert.assertNotEquals(hash1, hash2); + + // Reset and hash first again - should be same as original + hasher.reset(0); + hasher.update(data1); + Assert.assertEquals(hash1, hasher.getValue()); + } + + @Test + public void testHasherStreaming() { + // Test that streaming hasher produces same result as one-shot + byte[] data = "streaming test data for the hasher".getBytes(StandardCharsets.UTF_8); + + // One-shot + long oneShot = QwpSchemaHash.hash(data); + + // Streaming - byte by byte + QwpSchemaHash.Hasher hasher = new QwpSchemaHash.Hasher(); + hasher.reset(0); + for (byte b : data) { + hasher.update(b); + } + long streaming = hasher.getValue(); + + Assert.assertEquals("Streaming should match one-shot", oneShot, streaming); + } + + @Test + public void testHasherStreamingChunks() { + // Test streaming with various chunk sizes + byte[] data = "This is a longer test string to verify chunked hashing works correctly!".getBytes(StandardCharsets.UTF_8); + + long oneShot = QwpSchemaHash.hash(data); + + // Streaming - in chunks + QwpSchemaHash.Hasher hasher = new QwpSchemaHash.Hasher(); + hasher.reset(0); + + int pos = 0; + int[] chunkSizes = {5, 10, 3, 20, 7, 15}; + for (int chunkSize : chunkSizes) { + int toAdd = Math.min(chunkSize, data.length - pos); + if (toAdd > 0) { + hasher.update(data, pos, toAdd); + pos += toAdd; + } + } + // Add remaining + if (pos < data.length) { + hasher.update(data, pos, data.length - pos); + } + + Assert.assertEquals("Chunked streaming should match one-shot", oneShot, hasher.getValue()); + } + + @Test + public void testLargeSchema() { + // Test with many columns + int columnCount = 100; + String[] names = new String[columnCount]; + byte[] types = new byte[columnCount]; + + for (int i = 0; i < columnCount; i++) { + names[i] = "column_" + i; + types[i] = (byte) ((i % 15) + 1); // Cycle through types 1-15 + } + + long hash1 = QwpSchemaHash.computeSchemaHash(names, types); + long hash2 = QwpSchemaHash.computeSchemaHash(names, types); + + Assert.assertEquals("Large schema should hash consistently", hash1, hash2); + } + + @Test + public void testMultipleColumns() { + String[] names = {"symbol", "price", "timestamp"}; + byte[] types = {0x09, 0x07, 0x0A}; // SYMBOL, DOUBLE, TIMESTAMP + long hash = QwpSchemaHash.computeSchemaHash(names, types); + Assert.assertNotEquals(0, hash); + } + + @Test + public void testNameAffectsHash() { + // Different names, same type + byte[] types = {0x07}; // DOUBLE + + String[] names1 = {"price"}; + String[] names2 = {"value"}; + + long hash1 = QwpSchemaHash.computeSchemaHash(names1, types); + long hash2 = QwpSchemaHash.computeSchemaHash(names2, types); + + Assert.assertNotEquals("Name should affect hash", hash1, hash2); + } + + @Test + public void testDifferentTypeCodesProduceDifferentHashes() { + String[] names = {"value"}; + + byte[] types1 = {0x05}; // LONG + byte[] types2 = {0x06}; // DOUBLE + + long hash1 = QwpSchemaHash.computeSchemaHash(names, types1); + long hash2 = QwpSchemaHash.computeSchemaHash(names, types2); + + Assert.assertNotEquals("Different types should produce different hashes", hash1, hash2); + } + + @Test + public void testSchemaHashWithUtf8Names() { + // Test UTF-8 column names + String[] names = {"prix", "日時", "価格"}; // French, Japanese for datetime, Japanese for price + byte[] types = {0x07, 0x0A, 0x07}; + + long hash1 = QwpSchemaHash.computeSchemaHash(names, types); + long hash2 = QwpSchemaHash.computeSchemaHash(names, types); + + Assert.assertEquals("UTF-8 names should hash consistently", hash1, hash2); + Assert.assertNotEquals(0, hash1); + } + + @Test + public void testSingleColumn() { + String[] names = {"price"}; + byte[] types = {0x07}; // DOUBLE + long hash = QwpSchemaHash.computeSchemaHash(names, types); + Assert.assertNotEquals(0, hash); + } + + @Test + public void testTypeAffectsHash() { + // Same name, different type + String[] names = {"value"}; + + byte[] types1 = {0x04}; // INT + byte[] types2 = {0x05}; // LONG + + long hash1 = QwpSchemaHash.computeSchemaHash(names, types1); + long hash2 = QwpSchemaHash.computeSchemaHash(names, types2); + + Assert.assertNotEquals("Type should affect hash", hash1, hash2); + } + + @Test + public void testXXHash64DirectMemory() throws Exception { + assertMemoryLeak(() -> { + byte[] data = "test data".getBytes(StandardCharsets.UTF_8); + long addr = Unsafe.malloc(data.length, MemoryTag.NATIVE_ILP_RSS); + try { + for (int i = 0; i < data.length; i++) { + Unsafe.getUnsafe().putByte(addr + i, data[i]); + } + + long hashFromBytes = QwpSchemaHash.hash(data); + long hashFromMem = QwpSchemaHash.hash(addr, data.length); + + Assert.assertEquals("Direct memory hash should match byte array hash", hashFromBytes, hashFromMem); + } finally { + Unsafe.free(addr, data.length, MemoryTag.NATIVE_ILP_RSS); + } + }); + } + + @Test + public void testXXHash64Empty() { + byte[] data = new byte[0]; + long hash = QwpSchemaHash.hash(data); + // XXH64("", 0) = 0xEF46DB3751D8E999 + Assert.assertEquals(0xEF46DB3751D8E999L, hash); + } + + @Test + public void testXXHash64Exactly32Bytes() { + // Edge case: exactly 32 bytes + byte[] data = new byte[32]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte) (i & 0xFF); + } + + long hash = QwpSchemaHash.hash(data); + Assert.assertNotEquals(0, hash); + Assert.assertEquals(hash, QwpSchemaHash.hash(data)); + } + + @Test + public void testXXHash64KnownValue() { + // Test against a known XXHash64 value + // "abc" with seed 0 should produce a specific value + byte[] data = "abc".getBytes(StandardCharsets.UTF_8); + long hash = QwpSchemaHash.hash(data); + + // XXH64("abc", 0) = 0x44BC2CF5AD770999 + Assert.assertEquals(0x44BC2CF5AD770999L, hash); + } + + @Test + public void testXXHash64LongerString() { + // Test with a longer string to exercise the main loop + byte[] data = "Hello, World! This is a test string for XXHash64.".getBytes(StandardCharsets.UTF_8); + long hash1 = QwpSchemaHash.hash(data); + long hash2 = QwpSchemaHash.hash(data); + Assert.assertEquals(hash1, hash2); + Assert.assertNotEquals(0, hash1); + } + + @Test + public void testXXHash64Over32Bytes() { + // Test data longer than 32 bytes to exercise the main processing loop + byte[] data = new byte[100]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte) (i & 0xFF); + } + + long hash = QwpSchemaHash.hash(data); + Assert.assertNotEquals(0, hash); + + // Verify deterministic + Assert.assertEquals(hash, QwpSchemaHash.hash(data)); + } + + @Test + public void testXXHash64WithSeed() { + byte[] data = "test".getBytes(StandardCharsets.UTF_8); + + long hash0 = QwpSchemaHash.hash(data, 0, data.length, 0); + long hash1 = QwpSchemaHash.hash(data, 0, data.length, 1); + long hash42 = QwpSchemaHash.hash(data, 0, data.length, 42); + + // Different seeds should produce different hashes + Assert.assertNotEquals(hash0, hash1); + Assert.assertNotEquals(hash1, hash42); + Assert.assertNotEquals(hash0, hash42); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java new file mode 100644 index 0000000..24a3cd8 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java @@ -0,0 +1,1264 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.cairo.CairoException; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.line.array.DoubleArray; +import io.questdb.client.cutlass.line.array.LongArray; +import io.questdb.client.cutlass.qwp.protocol.OffHeapAppendMemory; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.Decimal128; +import io.questdb.client.std.Decimal64; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; +import static org.junit.Assert.*; + +public class QwpTableBufferTest { + + @Test + public void testAddDecimal128PrecisionLoss() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("d", QwpConstants.TYPE_DECIMAL128, true); + // First row sets decimalScale = 2 + col.addDecimal128(Decimal128.fromLong(100, 2)); + table.nextRow(); + // Second row at scale 4 with trailing fractional digits that + // cannot be represented at scale 2 without rounding + try { + col.addDecimal128(Decimal128.fromLong(12345, 4)); + fail("Expected LineSenderException for precision loss"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("precision loss")); + } + } + }); + } + + @Test + public void testAddDecimal128RescaleOverflow() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("d", QwpConstants.TYPE_DECIMAL128, true); + // First row sets decimalScale = 10 + col.addDecimal128(Decimal128.fromLong(1, 10)); + table.nextRow(); + // Second row at scale 0 with a large value — rescaling to scale 10 + // multiplies by 10^10, which exceeds 128-bit capacity + try { + col.addDecimal128(new Decimal128(Long.MAX_VALUE / 2, Long.MAX_VALUE, 0)); + fail("Expected LineSenderException for 128-bit overflow"); + } catch (LineSenderException e) { + assertEquals("Decimal128 overflow: rescaling from scale 0 to 10 exceeds 128-bit capacity", e.getMessage()); + } + } + }); + } + + @Test + public void testAddDecimal64PrecisionLoss() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("d", QwpConstants.TYPE_DECIMAL64, true); + // First row sets decimalScale = 2 + col.addDecimal64(Decimal64.fromLong(100, 2)); + table.nextRow(); + // Second row at scale 4 with trailing fractional digits that + // cannot be represented at scale 2 without rounding + try { + col.addDecimal64(Decimal64.fromLong(12345, 4)); + fail("Expected LineSenderException for precision loss"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("precision loss")); + } + } + }); + } + + @Test + public void testAddDecimal64RescaleOverflow() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("d", QwpConstants.TYPE_DECIMAL64, true); + // First row sets decimalScale = 5 + col.addDecimal64(Decimal64.fromLong(1, 5)); + table.nextRow(); + // Second row at scale 0 with a large value — rescaling to scale 5 + // multiplies by 10^5 = 100_000, which exceeds 64-bit capacity + // Long.MAX_VALUE / 10 ≈ 9.2 * 10^17, * 10^5 ≈ 9.2 * 10^22 >> 2^63 + try { + col.addDecimal64(Decimal64.fromLong(Long.MAX_VALUE / 10, 0)); + fail("Expected LineSenderException for 64-bit overflow"); + } catch (LineSenderException e) { + assertEquals("Decimal64 overflow: rescaling from scale 0 to 5 exceeds 64-bit capacity", e.getMessage()); + } + } + }); + } + + @Test + public void testAddDoubleArrayNullOnNonNullableColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); + + // Row 0: real array + col.addDoubleArray(new double[]{1.0, 2.0}); + table.nextRow(); + + // Row 1: null on non-nullable — must write empty array metadata + col.addDoubleArray((double[]) null); + table.nextRow(); + + // Row 2: real array + col.addDoubleArray(new double[]{3.0, 4.0}); + table.nextRow(); + + assertEquals(3, table.getRowCount()); + assertEquals(3, col.getValueCount()); + assertEquals(col.getSize(), col.getValueCount()); + + // Encoder walk must not corrupt — row 1 is an empty array + double[] encoded = readDoubleArraysLikeEncoder(col); + assertArrayEquals(new double[]{1.0, 2.0, 3.0, 4.0}, encoded, 0.0); + + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + assertEquals(1, dims[0]); + assertEquals(2, shapes[0]); + assertEquals(1, dims[1]); // null row: 1D empty + assertEquals(0, shapes[1]); // null row: 0 elements + assertEquals(1, dims[2]); + assertEquals(2, shapes[2]); + } + }); + } + + @Test + public void testAddDoubleArrayPayloadSupportsHigherDimensionalShape() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test"); + DoubleArray array = new DoubleArray(2, 1, 1, 2); + OffHeapAppendMemory payload = new OffHeapAppendMemory(128)) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); + + array.append(1.0).append(2.0).append(3.0).append(4.0); + array.appendToBufPtr(payload); + + col.addDoubleArrayPayload(payload.pageAddress(), payload.getAppendOffset()); + table.nextRow(); + + assertEquals(1, col.getValueCount()); + + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + assertEquals(4, dims[0]); + assertEquals(2, shapes[0]); + assertEquals(1, shapes[1]); + assertEquals(1, shapes[2]); + assertEquals(2, shapes[3]); + + assertArrayEquals(new double[]{1.0, 2.0, 3.0, 4.0}, readDoubleArraysLikeEncoder(col), 0.0); + } + }); + } + + @Test + public void testAddLongArrayNullOnNonNullableColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false); + + // Row 0: real array + col.addLongArray(new long[]{10, 20}); + table.nextRow(); + + // Row 1: null on non-nullable — must write empty array metadata + col.addLongArray((long[]) null); + table.nextRow(); + + // Row 2: real array + col.addLongArray(new long[]{30, 40}); + table.nextRow(); + + assertEquals(3, table.getRowCount()); + assertEquals(3, col.getValueCount()); + assertEquals(col.getSize(), col.getValueCount()); + + // Encoder walk must not corrupt — row 1 is an empty array + long[] encoded = readLongArraysLikeEncoder(col); + assertArrayEquals(new long[]{10, 20, 30, 40}, encoded); + + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + assertEquals(1, dims[0]); + assertEquals(2, shapes[0]); + assertEquals(1, dims[1]); // null row: 1D empty + assertEquals(0, shapes[1]); // null row: 0 elements + assertEquals(1, dims[2]); + assertEquals(2, shapes[2]); + } + }); + } + + @Test + public void testAddSymbolNullOnNonNullableColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("sym", QwpConstants.TYPE_SYMBOL, false); + col.addSymbol("server1"); + table.nextRow(); + + // Null on a non-nullable column must write a sentinel value, + // keeping size and valueCount in sync + col.addSymbol(null); + table.nextRow(); + + col.addSymbol("server2"); + table.nextRow(); + + assertEquals(3, table.getRowCount()); + // For non-nullable columns, every row must have a physical value + assertEquals(col.getSize(), col.getValueCount()); + } + }); + } + + @Test + public void testAddSymbolUtf8CancelRowRewindsDictionary() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(0); + table.nextRow(); + + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("s", QwpConstants.TYPE_SYMBOL, true); + addSymbolUtf8(col, "stale"); + table.cancelCurrentRow(); + + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); + addSymbolUtf8(col, "fresh"); + table.nextRow(); + + assertEquals(2, col.getSize()); + assertEquals(1, col.getValueCount()); + assertArrayEquals(new String[]{"fresh"}, col.getSymbolDictionary()); + assertEquals(0, Unsafe.getUnsafe().getInt(col.getDataAddress())); + } + }); + } + + @Test + public void testAddSymbolUtf8RejectsInvalidUtf8() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("sym", QwpConstants.TYPE_SYMBOL, true); + byte[] invalid = {(byte) 0xC3, 0x28}; + long ptr = copyToNative(invalid); + try { + try { + col.addSymbolUtf8(ptr, invalid.length); + fail("Expected CairoException"); + } catch (CairoException ex) { + assertTrue(ex.getFlyweightMessage().toString().contains("cannot convert invalid UTF-8 sequence")); + } + assertEquals(0, col.getSize()); + assertEquals(0, col.getValueCount()); + assertEquals(0, col.getSymbolDictionarySize()); + } finally { + Unsafe.free(ptr, invalid.length, MemoryTag.NATIVE_DEFAULT); + } + } + }); + } + + @Test + public void testAddSymbolUtf8ReusesExistingDictionaryEntry() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("sym", QwpConstants.TYPE_SYMBOL, true); + addSymbolUtf8(col, "東京"); + table.nextRow(); + + addSymbolUtf8(col, "東京"); + table.nextRow(); + + addSymbolUtf8(col, "Αθηνα"); + table.nextRow(); + + assertEquals(3, col.getSize()); + assertEquals(3, col.getValueCount()); + assertEquals(2, col.getSymbolDictionarySize()); + assertArrayEquals(new String[]{"東京", "Αθηνα"}, col.getSymbolDictionary()); + + long dataAddress = col.getDataAddress(); + assertEquals(0, Unsafe.getUnsafe().getInt(dataAddress)); + assertEquals(0, Unsafe.getUnsafe().getInt(dataAddress + 4)); + assertEquals(1, Unsafe.getUnsafe().getInt(dataAddress + 8)); + } + }); + } + + @Test + public void testAddSymbolWithGlobalIdStoresOnlyGlobalIds() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("sym", QwpConstants.TYPE_SYMBOL, true); + col.addSymbolWithGlobalId("alpha", 7); + table.nextRow(); + col.addSymbolWithGlobalId("beta", 11); + table.nextRow(); + + assertEquals(2, col.getSize()); + assertEquals(2, col.getValueCount()); + assertEquals(0, col.getSymbolDictionarySize()); + assertEquals(0, col.getAuxDataAddress()); + assertEquals(11, col.getMaxGlobalSymbolId()); + + long dataAddress = col.getDataAddress(); + assertEquals(7, Unsafe.getUnsafe().getInt(dataAddress)); + assertEquals(11, Unsafe.getUnsafe().getInt(dataAddress + Integer.BYTES)); + } + }); + } + + @Test + public void testCancelRowResetsDecimalScaleOnLateAddedColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(0); + table.nextRow(); + + // Start row 2: create a decimal column with scale 5 + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); + QwpTableBuffer.ColumnBuffer colD = table.getOrCreateColumn("d", QwpConstants.TYPE_DECIMAL64, true); + colD.addDecimal64(Decimal64.fromLong(100, 5)); + table.cancelCurrentRow(); + + // After cancel, decimalScale must be reset. Adding a value at scale 3 + // should succeed and use scale 3 as the column's scale. + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); + colD.addDecimal64(Decimal64.fromLong(42, 3)); + table.nextRow(); + + assertEquals(2, colD.getSize()); + assertEquals(1, colD.getValueCount()); + } + }); + } + + @Test + public void testCancelRowResetsGeohashPrecisionOnLateAddedColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(0); + table.nextRow(); + + // Start row 2: create a geohash column with 20-bit precision + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); + QwpTableBuffer.ColumnBuffer colG = table.getOrCreateColumn("g", QwpConstants.TYPE_GEOHASH, true); + colG.addGeoHash(123L, 20); + table.cancelCurrentRow(); + + // After cancel, geohashPrecision must be reset. Adding a value at + // 30-bit precision should succeed without a precision mismatch error. + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); + colG.addGeoHash(456L, 30); + table.nextRow(); + + assertEquals(2, colG.getSize()); + assertEquals(1, colG.getValueCount()); + } + }); + } + + @Test + public void testCancelRowResetsSymbolDictOnLateAddedColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(0); + table.nextRow(); + + // Start row 2: create a symbol column with value "stale" + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); + QwpTableBuffer.ColumnBuffer colS = table.getOrCreateColumn("s", QwpConstants.TYPE_SYMBOL, true); + colS.addSymbol("stale"); + table.cancelCurrentRow(); + + // After cancel, symbol dictionary must be empty. + // "fresh" should get local ID 0, not 1. + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); + colS.addSymbol("fresh"); + table.nextRow(); + + assertEquals(2, colS.getSize()); + assertEquals(1, colS.getValueCount()); + String[] dict = colS.getSymbolDictionary(); + assertEquals(1, dict.length); + assertEquals("fresh", dict[0]); + } + }); + } + + @Test + public void testCancelRowRetainsGlobalSymbolIdWithoutLocalDictionary() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(0); + table.nextRow(); + + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); + QwpTableBuffer.ColumnBuffer colS = table.getOrCreateColumn("s", QwpConstants.TYPE_SYMBOL, true); + colS.addSymbolWithGlobalId("stale", 4); + table.cancelCurrentRow(); + + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); + colS.addSymbolWithGlobalId("fresh", 9); + table.nextRow(); + + assertEquals(2, colS.getSize()); + assertEquals(1, colS.getValueCount()); + assertEquals(0, colS.getSymbolDictionarySize()); + assertEquals(9, colS.getMaxGlobalSymbolId()); + assertEquals(9, Unsafe.getUnsafe().getInt(colS.getDataAddress())); + } + }); + } + + @Test + public void testCancelRowRewindsDoubleArrayOffsets() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + // Row 0: committed with [1.0, 2.0] + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); + col.addDoubleArray(new double[]{1.0, 2.0}); + table.nextRow(); + + // Row 1: committed with [3.0, 4.0] + col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); + col.addDoubleArray(new double[]{3.0, 4.0}); + table.nextRow(); + + // Start row 2 with [5.0, 6.0] — then cancel it + col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); + col.addDoubleArray(new double[]{5.0, 6.0}); + table.cancelCurrentRow(); + + // Add replacement row 2 with [7.0, 8.0] + col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); + col.addDoubleArray(new double[]{7.0, 8.0}); + table.nextRow(); + + assertEquals(3, table.getRowCount()); + assertEquals(3, col.getValueCount()); + + // Walk the arrays exactly as the encoder would + double[] encoded = readDoubleArraysLikeEncoder(col); + assertArrayEquals( + new double[]{1.0, 2.0, 3.0, 4.0, 7.0, 8.0}, + encoded, + 0.0 + ); + } + }); + } + + @Test + public void testCancelRowRewindsLongArrayOffsets() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + // Row 0: committed with [10, 20] + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false); + col.addLongArray(new long[]{10, 20}); + table.nextRow(); + + // Start row 1 with [30, 40] — then cancel it + col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false); + col.addLongArray(new long[]{30, 40}); + table.cancelCurrentRow(); + + // Add replacement row 1 with [50, 60] + col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false); + col.addLongArray(new long[]{50, 60}); + table.nextRow(); + + assertEquals(2, table.getRowCount()); + assertEquals(2, col.getValueCount()); + + long[] encoded = readLongArraysLikeEncoder(col); + assertArrayEquals(new long[]{10, 20, 50, 60}, encoded); + } + }); + } + + @Test + public void testCancelRowRewindsMultiDimArrayOffsets() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + // Row 0: committed 2D array [[1.0, 2.0], [3.0, 4.0]] + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); + col.addDoubleArray(new double[][]{{1.0, 2.0}, {3.0, 4.0}}); + table.nextRow(); + + // Start row 1 with 2D array [[5.0, 6.0], [7.0, 8.0]] — cancel + col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); + col.addDoubleArray(new double[][]{{5.0, 6.0}, {7.0, 8.0}}); + table.cancelCurrentRow(); + + // Replacement row 1 with [[9.0, 10.0], [11.0, 12.0]] + col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); + col.addDoubleArray(new double[][]{{9.0, 10.0}, {11.0, 12.0}}); + table.nextRow(); + + assertEquals(2, table.getRowCount()); + assertEquals(2, col.getValueCount()); + + // Verify shapes are correct (2 dims per row, each [2, 2]) + int[] shapes = col.getArrayShapes(); + byte[] dims = col.getArrayDims(); + assertEquals(2, dims[0]); + assertEquals(2, dims[1]); + // Row 0 shapes: [2, 2] + assertEquals(2, shapes[0]); + assertEquals(2, shapes[1]); + // Row 1 shapes must be the replacement [2, 2], not stale data + assertEquals(2, shapes[2]); + assertEquals(2, shapes[3]); + + double[] encoded = readDoubleArraysLikeEncoder(col); + assertArrayEquals( + new double[]{1.0, 2.0, 3.0, 4.0, 9.0, 10.0, 11.0, 12.0}, + encoded, + 0.0 + ); + } + }); + } + + @Test + public void testCancelRowTruncatesLateAddedColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + // Commit 3 rows with columns "a" (LONG, non-nullable) and "b" (STRING, nullable) + for (int i = 0; i < 3; i++) { + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(i); + table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true).addString("v" + i); + table.nextRow(); + } + + // Start row 4: set "a" and "b", then create a NEW column "c" + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(3); + table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true).addString("v3"); + QwpTableBuffer.ColumnBuffer colC = table.getOrCreateColumn("c", QwpConstants.TYPE_STRING, true); + colC.addString("stale"); + + // Cancel the in-progress row + table.cancelCurrentRow(); + + // Column "c" was created during the in-progress row, so it must be fully cleared + assertEquals(0, colC.getSize()); + assertEquals(0, colC.getValueCount()); + + // Start row 4 again: set "a" and "b" only (not "c") + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(3); + table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true).addString("v3"); + table.nextRow(); + + // Column "c" should now have size == 4 (padded with nulls) and valueCount == 0 + assertEquals(4, colC.getSize()); + assertEquals(0, colC.getValueCount()); + + // All 4 rows of column "c" should be null + for (int i = 0; i < 4; i++) { + assertTrue("row " + i + " of column c should be null", colC.isNull(i)); + } + } + }); + } + + @Test + public void testCancelRowTruncatesLateAddedColumnWhenSizeEqualsRowCount() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + // Commit exactly 1 row so rowCount == 1 + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(0); + table.nextRow(); + + // Start row 2: set "a", then create NEW column "c" with one value + // col_c.size will be 1, which equals rowCount — the edge case + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); + QwpTableBuffer.ColumnBuffer colC = table.getOrCreateColumn("c", QwpConstants.TYPE_STRING, true); + colC.addString("stale"); + + // Cancel the in-progress row + table.cancelCurrentRow(); + + // Column "c" had size == rowCount (1 == 1) but was still late-added + assertEquals(0, colC.getSize()); + assertEquals(0, colC.getValueCount()); + + // Start row 2 again without setting "c" + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); + table.nextRow(); + + // Column "c" should have 2 null rows + assertEquals(2, colC.getSize()); + assertEquals(0, colC.getValueCount()); + assertTrue(colC.isNull(0)); + assertTrue(colC.isNull(1)); + } + }); + } + + @Test + public void testDoubleArrayWrapperMultipleRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test"); + DoubleArray arr = new DoubleArray(3)) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); + + arr.append(1.0).append(2.0).append(3.0); + col.addDoubleArray(arr); + table.nextRow(); + + // DoubleArray auto-wraps, so just append next row's data + arr.append(4.0).append(5.0).append(6.0); + col.addDoubleArray(arr); + table.nextRow(); + + arr.append(7.0).append(8.0).append(9.0); + col.addDoubleArray(arr); + table.nextRow(); + + assertEquals(3, col.getValueCount()); + double[] encoded = readDoubleArraysLikeEncoder(col); + assertArrayEquals( + new double[]{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0}, + encoded, + 0.0 + ); + + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + for (int i = 0; i < 3; i++) { + assertEquals(1, dims[i]); + assertEquals(3, shapes[i]); + } + } + }); + } + + @Test + public void testDoubleArrayWrapperShrinkingSize() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); + + // Row 0: large array (5 elements) + try (DoubleArray big = new DoubleArray(5)) { + big.append(1.0).append(2.0).append(3.0).append(4.0).append(5.0); + col.addDoubleArray(big); + table.nextRow(); + } + + // Row 1: smaller array (2 elements) — must not see leftover data from row 0 + try (DoubleArray small = new DoubleArray(2)) { + small.append(10.0).append(20.0); + col.addDoubleArray(small); + table.nextRow(); + } + + assertEquals(2, col.getValueCount()); + double[] encoded = readDoubleArraysLikeEncoder(col); + assertArrayEquals( + new double[]{1.0, 2.0, 3.0, 4.0, 5.0, 10.0, 20.0}, + encoded, + 0.0 + ); + + int[] shapes = col.getArrayShapes(); + assertEquals(5, shapes[0]); + assertEquals(2, shapes[1]); + } + }); + } + + @Test + public void testDoubleArrayWrapperVaryingDimensionality() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); + + // Row 0: 2D array (2x2) + try (DoubleArray matrix = new DoubleArray(2, 2)) { + matrix.append(1.0).append(2.0).append(3.0).append(4.0); + col.addDoubleArray(matrix); + table.nextRow(); + } + + // Row 1: 1D array (3 elements) — different dimensionality + try (DoubleArray vec = new DoubleArray(3)) { + vec.append(10.0).append(20.0).append(30.0); + col.addDoubleArray(vec); + table.nextRow(); + } + + assertEquals(2, col.getValueCount()); + + byte[] dims = col.getArrayDims(); + assertEquals(2, dims[0]); + assertEquals(1, dims[1]); + + int[] shapes = col.getArrayShapes(); + // Row 0: shape [2, 2] + assertEquals(2, shapes[0]); + assertEquals(2, shapes[1]); + // Row 1: shape [3] + assertEquals(3, shapes[2]); + + double[] encoded = readDoubleArraysLikeEncoder(col); + assertArrayEquals( + new double[]{1.0, 2.0, 3.0, 4.0, 10.0, 20.0, 30.0}, + encoded, + 0.0 + ); + } + }); + } + + @Test + public void testGetExistingColumnReturnsNullWithoutCreatingColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer colA = table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false); + colA.addLong(1); + table.nextRow(); + + assertNull(table.getExistingColumn("missing", QwpConstants.TYPE_STRING)); + assertEquals(1, table.getColumnCount()); + + QwpTableBuffer.ColumnBuffer colB = table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true); + assertNotNull(colB); + assertEquals(2, table.getColumnCount()); + } + }); + } + + @Test + public void testGetExistingColumnReturnsOrderedColumnsAcrossRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer colA = table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false); + QwpTableBuffer.ColumnBuffer colB = table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true); + colA.addLong(1); + colB.addString("x"); + table.nextRow(); + + QwpTableBuffer.ColumnBuffer existingA = table.getExistingColumn("a", QwpConstants.TYPE_LONG); + QwpTableBuffer.ColumnBuffer existingB = table.getExistingColumn("b", QwpConstants.TYPE_STRING); + + assertSame(colA, existingA); + assertSame(colB, existingB); + + existingA.addLong(2); + existingB.addString("y"); + table.nextRow(); + + assertEquals(2, table.getRowCount()); + assertEquals(2, colA.getSize()); + assertEquals(2, colA.getValueCount()); + assertEquals(2, colB.getSize()); + assertEquals(2, colB.getValueCount()); + } + }); + } + + @Test + public void testGetExistingColumnReturnsOutOfOrderColumns() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer colA = table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false); + QwpTableBuffer.ColumnBuffer colB = table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true); + colA.addLong(1); + colB.addString("x"); + table.nextRow(); + + QwpTableBuffer.ColumnBuffer existingB = table.getExistingColumn("b", QwpConstants.TYPE_STRING); + QwpTableBuffer.ColumnBuffer existingA = table.getExistingColumn("a", QwpConstants.TYPE_LONG); + + assertSame(colB, existingB); + assertSame(colA, existingA); + + existingB.addString("y"); + existingA.addLong(2); + table.nextRow(); + + assertEquals(2, table.getRowCount()); + assertEquals(2, colA.getSize()); + assertEquals(2, colA.getValueCount()); + assertEquals(2, colB.getSize()); + assertEquals(2, colB.getValueCount()); + } + }); + } + + @Test + public void testGetExistingColumnTypeMismatchOnHashPathThrows() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer colA = table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false); + QwpTableBuffer.ColumnBuffer colB = table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true); + colA.addLong(1); + colB.addString("x"); + table.nextRow(); + + try { + table.getExistingColumn("b", QwpConstants.TYPE_LONG); + fail("Expected LineSenderException for hash-path type mismatch"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("Column type mismatch")); + assertTrue(e.getMessage().contains("column 'b'")); + } + } + }); + } + + @Test + public void testGetExistingColumnTypeMismatchOnOrderedPathThrows() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer colA = table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false); + table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true); + colA.addLong(1); + table.nextRow(); + + try { + table.getExistingColumn("a", QwpConstants.TYPE_STRING); + fail("Expected LineSenderException for ordered-path type mismatch"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("Column type mismatch")); + assertTrue(e.getMessage().contains("column 'a'")); + } + } + }); + } + + @Test + public void testGetExistingColumnWorksAfterReset() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer colA = table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false); + QwpTableBuffer.ColumnBuffer colB = table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true); + colA.addLong(1); + colB.addString("x"); + table.nextRow(); + + table.reset(); + + QwpTableBuffer.ColumnBuffer existingA = table.getExistingColumn("a", QwpConstants.TYPE_LONG); + QwpTableBuffer.ColumnBuffer existingB = table.getExistingColumn("b", QwpConstants.TYPE_STRING); + + assertSame(colA, existingA); + assertSame(colB, existingB); + + existingA.addLong(2); + existingB.addString("y"); + table.nextRow(); + + assertEquals(1, table.getRowCount()); + assertEquals(1, colA.getSize()); + assertEquals(1, colA.getValueCount()); + assertEquals(1, colB.getSize()); + assertEquals(1, colB.getValueCount()); + } + }); + } + + @Test + public void testGetExistingColumnWorksForLateAddedColumnAfterCancelRow() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); + table.nextRow(); + + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(2); + QwpTableBuffer.ColumnBuffer late = table.getOrCreateColumn("late", QwpConstants.TYPE_STRING, true); + late.addString("stale"); + table.cancelCurrentRow(); + + QwpTableBuffer.ColumnBuffer existingLate = table.getExistingColumn("late", QwpConstants.TYPE_STRING); + assertSame(late, existingLate); + assertEquals(0, existingLate.getSize()); + assertEquals(0, existingLate.getValueCount()); + + table.getExistingColumn("a", QwpConstants.TYPE_LONG).addLong(2); + table.nextRow(); + + assertEquals(2, table.getRowCount()); + assertEquals(2, existingLate.getSize()); + assertEquals(0, existingLate.getValueCount()); + assertTrue(existingLate.isNull(0)); + assertTrue(existingLate.isNull(1)); + } + }); + } + + @Test + public void testGetOrCreateColumnConflictingTypeFastPath() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + // First call creates the column as LONG + table.getOrCreateColumn("x", QwpConstants.TYPE_LONG, false).addLong(1L); + table.nextRow(); + + // Second call with the same name but a different type hits the fast path + // (sequential cursor matches the column name) and must throw + try { + table.getOrCreateColumn("x", QwpConstants.TYPE_DOUBLE, false); + fail("Expected LineSenderException for column type mismatch"); + } catch (LineSenderException e) { + assertEquals( + "Column type mismatch for column 'x': columnType=" + QwpConstants.TYPE_LONG + ", sentType=" + QwpConstants.TYPE_DOUBLE, + e.getMessage() + ); + } + } + }); + } + + @Test + public void testGetOrCreateColumnConflictingTypeSlowPath() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + // Create two columns so the fast-path cursor can be defeated + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1L); + table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true).addString("v"); + table.nextRow(); + + // Access column "b" first — cursor now expects "a" at index 0, + // but we ask for "b", so the fast path misses and falls through + // to the hash-map lookup, which must detect the type conflict + try { + table.getOrCreateColumn("b", QwpConstants.TYPE_LONG, false); + fail("Expected LineSenderException for column type mismatch"); + } catch (LineSenderException e) { + assertEquals( + "Column type mismatch for column 'b': columnType=" + QwpConstants.TYPE_STRING + ", sentType=" + QwpConstants.TYPE_LONG, + e.getMessage() + ); + } + } + }); + } + + @Test + public void testLongArrayMultipleRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false); + + col.addLongArray(new long[]{10, 20, 30}); + table.nextRow(); + + col.addLongArray(new long[]{40, 50, 60}); + table.nextRow(); + + col.addLongArray(new long[]{70, 80, 90}); + table.nextRow(); + + assertEquals(3, col.getValueCount()); + long[] encoded = readLongArraysLikeEncoder(col); + assertArrayEquals( + new long[]{10, 20, 30, 40, 50, 60, 70, 80, 90}, + encoded + ); + + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + for (int i = 0; i < 3; i++) { + assertEquals(1, dims[i]); + assertEquals(3, shapes[i]); + } + } + }); + } + + @Test + public void testLongArrayShrinkingSize() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false); + + // Row 0: large array (4 elements) + col.addLongArray(new long[]{100, 200, 300, 400}); + table.nextRow(); + + // Row 1: smaller array (2 elements) — must not see leftover data from row 0 + col.addLongArray(new long[]{10, 20}); + table.nextRow(); + + assertEquals(2, col.getValueCount()); + long[] encoded = readLongArraysLikeEncoder(col); + assertArrayEquals(new long[]{100, 200, 300, 400, 10, 20}, encoded); + + int[] shapes = col.getArrayShapes(); + assertEquals(4, shapes[0]); + assertEquals(2, shapes[1]); + } + }); + } + + @Test + public void testLongArrayWrapperMultipleRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test"); + LongArray arr = new LongArray(3)) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false); + + arr.append(10).append(20).append(30); + col.addLongArray(arr); + table.nextRow(); + + arr.append(40).append(50).append(60); + col.addLongArray(arr); + table.nextRow(); + + arr.append(70).append(80).append(90); + col.addLongArray(arr); + table.nextRow(); + + assertEquals(3, col.getValueCount()); + long[] encoded = readLongArraysLikeEncoder(col); + assertArrayEquals(new long[]{10, 20, 30, 40, 50, 60, 70, 80, 90}, encoded); + + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + for (int i = 0; i < 3; i++) { + assertEquals(1, dims[i]); + assertEquals(3, shapes[i]); + } + } + }); + } + + @Test + public void testNextRowWithPreparedMissingColumnsPadsListedColumns() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer colA = table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false); + QwpTableBuffer.ColumnBuffer colB = table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true); + QwpTableBuffer.ColumnBuffer colC = table.getOrCreateColumn("c", QwpConstants.TYPE_LONG, false); + + colA.addLong(10); + colB.addString("x"); + colC.addLong(100); + table.nextRow(); + + colA.addLong(20); + table.nextRow(new QwpTableBuffer.ColumnBuffer[]{colB, colC}, 2); + + assertEquals(2, colA.getSize()); + assertEquals(2, colA.getValueCount()); + assertEquals(2, colB.getSize()); + assertEquals(1, colB.getValueCount()); + assertFalse(colB.isNull(0)); + assertTrue(colB.isNull(1)); + assertEquals(2, colC.getSize()); + assertEquals(2, colC.getValueCount()); + assertEquals(100L, Unsafe.getUnsafe().getLong(colC.getDataAddress())); + assertEquals(Long.MIN_VALUE, Unsafe.getUnsafe().getLong(colC.getDataAddress() + Long.BYTES)); + } + }); + } + + @Test + public void testRetainInProgressRowFastClearsUnstagedNullableColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer keep = table.getOrCreateColumn("keep", QwpConstants.TYPE_LONG, false); + QwpTableBuffer.ColumnBuffer drop = table.getOrCreateColumn("drop", QwpConstants.TYPE_STRING, true); + + for (int i = 0; i < 130; i++) { + keep.addLong(i); + if ((i & 1) == 0) { + drop.addString("v" + i); + } else { + drop.addNull(); + } + table.nextRow(); + } + + int keepSizeBefore = keep.getSize(); + int keepValueCountBefore = keep.getValueCount(); + long keepStringDataSizeBefore = keep.getStringDataSize(); + int keepArrayShapeOffsetBefore = keep.getArrayShapeOffset(); + int keepArrayDataOffsetBefore = keep.getArrayDataOffset(); + int keepIndex = keep.getIndex(); + + keep.addLong(130); + + int[] sizeBefore = {-1, -1}; + int[] valueCountBefore = {-1, -1}; + long[] stringDataSizeBefore = new long[2]; + int[] arrayShapeOffsetBefore = new int[2]; + int[] arrayDataOffsetBefore = new int[2]; + + sizeBefore[keepIndex] = keepSizeBefore; + valueCountBefore[keepIndex] = keepValueCountBefore; + stringDataSizeBefore[keepIndex] = keepStringDataSizeBefore; + arrayShapeOffsetBefore[keepIndex] = keepArrayShapeOffsetBefore; + arrayDataOffsetBefore[keepIndex] = keepArrayDataOffsetBefore; + + table.retainInProgressRow( + sizeBefore, + valueCountBefore, + arrayShapeOffsetBefore, + arrayDataOffsetBefore + ); + + assertEquals(0, table.getRowCount()); + + assertEquals(1, keep.getSize()); + assertEquals(1, keep.getValueCount()); + assertEquals(130L, Unsafe.getUnsafe().getLong(keep.getDataAddress())); + + assertEquals(0, drop.getSize()); + assertEquals(0, drop.getValueCount()); + assertEquals(0, drop.getStringDataSize()); + assertFalse(drop.isNull(0)); + assertFalse(drop.isNull(63)); + assertFalse(drop.isNull(64)); + assertFalse(drop.isNull(129)); + assertEquals(0, Unsafe.getUnsafe().getLong(drop.getNullBitmapAddress())); + assertEquals(0, Unsafe.getUnsafe().getLong(drop.getNullBitmapAddress() + Long.BYTES)); + assertEquals(0, Unsafe.getUnsafe().getLong(drop.getNullBitmapAddress() + 2L * Long.BYTES)); + assertEquals(0, Unsafe.getUnsafe().getInt(drop.getStringOffsetsAddress())); + } + }); + } + + private static void addSymbolUtf8(QwpTableBuffer.ColumnBuffer col, String value) { + byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + long ptr = copyToNative(bytes); + try { + col.addSymbolUtf8(ptr, bytes.length); + } finally { + Unsafe.free(ptr, bytes.length, MemoryTag.NATIVE_DEFAULT); + } + } + + private static long copyToNative(byte[] bytes) { + long ptr = Unsafe.malloc(bytes.length, MemoryTag.NATIVE_DEFAULT); + for (int i = 0; i < bytes.length; i++) { + Unsafe.getUnsafe().putByte(ptr + i, bytes[i]); + } + return ptr; + } + + /** + * Simulates the encoder's walk over array data — the same logic as + * QwpWebSocketEncoder.writeDoubleArrayColumn(). Returns the flat + * double values the encoder would serialize for the given column. + */ + private static double[] readDoubleArraysLikeEncoder(QwpTableBuffer.ColumnBuffer col) { + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + double[] data = col.getDoubleArrayData(); + int count = col.getValueCount(); + + // First pass: count total elements + int totalElements = 0; + int shapeIdx = 0; + for (int row = 0; row < count; row++) { + int nDims = dims[row]; + int elemCount = 1; + for (int d = 0; d < nDims; d++) { + elemCount *= shapes[shapeIdx++]; + } + totalElements += elemCount; + } + + // Second pass: collect values + double[] result = new double[totalElements]; + shapeIdx = 0; + int dataIdx = 0; + int resultIdx = 0; + for (int row = 0; row < count; row++) { + int nDims = dims[row]; + int elemCount = 1; + for (int d = 0; d < nDims; d++) { + elemCount *= shapes[shapeIdx++]; + } + for (int e = 0; e < elemCount; e++) { + result[resultIdx++] = data[dataIdx++]; + } + } + return result; + } + + /** + * Same as above but for long arrays (mirrors QwpWebSocketEncoder.writeLongArrayColumn()). + */ + private static long[] readLongArraysLikeEncoder(QwpTableBuffer.ColumnBuffer col) { + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + long[] data = col.getLongArrayData(); + int count = col.getValueCount(); + + int totalElements = 0; + int shapeIdx = 0; + for (int row = 0; row < count; row++) { + int nDims = dims[row]; + int elemCount = 1; + for (int d = 0; d < nDims; d++) { + elemCount *= shapes[shapeIdx++]; + } + totalElements += elemCount; + } + + long[] result = new long[totalElements]; + shapeIdx = 0; + int dataIdx = 0; + int resultIdx = 0; + for (int row = 0; row < count; row++) { + int nDims = dims[row]; + int elemCount = 1; + for (int d = 0; d < nDims; d++) { + elemCount *= shapes[shapeIdx++]; + } + for (int e = 0; e < elemCount; e++) { + result[resultIdx++] = data[dataIdx++]; + } + } + return result; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java new file mode 100644 index 0000000..79ef4ce --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java @@ -0,0 +1,395 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.websocket; + +import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; +import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.SocketTimeoutException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.Base64; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * A simple WebSocket server for client integration testing. + * Uses plain Java heap buffers - no native memory. + */ +public class TestWebSocketServer implements Closeable { + private static final Logger LOG = LoggerFactory.getLogger(TestWebSocketServer.class); + private static final String WEBSOCKET_GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; + private final List clients = new CopyOnWriteArrayList<>(); + private final WebSocketServerHandler handler; + private final int port; + private final AtomicBoolean running = new AtomicBoolean(false); + private final CountDownLatch startLatch = new CountDownLatch(1); + private Thread acceptThread; + private ServerSocket serverSocket; + + public TestWebSocketServer(int port, WebSocketServerHandler handler) { + this.port = port; + this.handler = handler; + } + + public boolean awaitStart(long timeout, TimeUnit unit) throws InterruptedException { + return startLatch.await(timeout, unit); + } + + @Override + public void close() { + running.set(false); + + for (ClientHandler client : clients) { + client.close(); + } + clients.clear(); + + if (serverSocket != null) { + try { + serverSocket.close(); + } catch (IOException e) { + // ignore + } + } + + if (acceptThread != null) { + try { + acceptThread.join(5000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } + + public void start() throws IOException { + if (running.getAndSet(true)) { + return; + } + + serverSocket = new ServerSocket(port); + serverSocket.setSoTimeout(100); + + acceptThread = new Thread(() -> { + startLatch.countDown(); + while (running.get()) { + try { + Socket clientSocket = serverSocket.accept(); + ClientHandler clientHandler = new ClientHandler(clientSocket); + clients.add(clientHandler); + clientHandler.start(); + } catch (SocketTimeoutException e) { + // expected, check running flag + } catch (IOException e) { + if (running.get()) { + LOG.error("Accept error", e); + } + } + } + }, "WebSocket-Accept"); + acceptThread.start(); + } + + /** + * Interface for handling WebSocket server events. + */ + public interface WebSocketServerHandler { + default void onBinaryMessage(ClientHandler client, byte[] data) { + } + } + + /** + * Handles a single WebSocket client connection. + */ + public class ClientHandler implements Closeable { + private final ByteBuffer recvBuffer = ByteBuffer.allocate(65_536).order(ByteOrder.BIG_ENDIAN); + private final AtomicBoolean running = new AtomicBoolean(false); + private final Socket socket; + private InputStream in; + private boolean isClosed; + private OutputStream out; + private Thread readThread; + + ClientHandler(Socket socket) { + this.socket = socket; + recvBuffer.flip(); // start with nothing readable + } + + @Override + public void close() { + running.set(false); + try { + socket.close(); + } catch (IOException e) { + // ignore + } + if (readThread != null) { + try { + readThread.join(5000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } + + public synchronized void sendBinary(byte[] data) throws IOException { + writeFrame(WebSocketOpcode.BINARY, data, data.length); + } + + public synchronized void sendClose(int code, String reason) throws IOException { + byte[] reasonBytes = (reason != null && !reason.isEmpty()) + ? reason.getBytes(StandardCharsets.UTF_8) : new byte[0]; + byte[] payload = new byte[2 + reasonBytes.length]; + payload[0] = (byte) ((code >> 8) & 0xFF); + payload[1] = (byte) (code & 0xFF); + System.arraycopy(reasonBytes, 0, payload, 2, reasonBytes.length); + writeFrame(WebSocketOpcode.CLOSE, payload, payload.length); + } + + public synchronized void sendPing(byte[] data) throws IOException { + writeFrame(WebSocketOpcode.PING, data, data.length); + } + + private String computeAcceptKey(String key) { + try { + MessageDigest sha1 = MessageDigest.getInstance("SHA-1"); + sha1.update((key + WEBSOCKET_GUID).getBytes(StandardCharsets.US_ASCII)); + return Base64.getEncoder().encodeToString(sha1.digest()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void handleRead() { + while (recvBuffer.remaining() >= 2) { + recvBuffer.mark(); + + int byte0 = recvBuffer.get() & 0xFF; + int byte1 = recvBuffer.get() & 0xFF; + + int opcode = byte0 & 0x0F; + boolean isMasked = (byte1 & 0x80) != 0; + int lengthField = byte1 & 0x7F; + + long payloadLength; + if (lengthField <= 125) { + payloadLength = lengthField; + } else if (lengthField == 126) { + if (recvBuffer.remaining() < 2) { + recvBuffer.reset(); + return; + } + payloadLength = (recvBuffer.get() & 0xFF) << 8 | (recvBuffer.get() & 0xFF); + } else { + if (recvBuffer.remaining() < 8) { + recvBuffer.reset(); + return; + } + payloadLength = recvBuffer.getLong(); + } + + int maskKeySize = isMasked ? 4 : 0; + if (recvBuffer.remaining() < maskKeySize + payloadLength) { + recvBuffer.reset(); + return; + } + + byte[] maskKey = null; + if (isMasked) { + maskKey = new byte[4]; + recvBuffer.get(maskKey); + } + + byte[] payload = new byte[(int) payloadLength]; + recvBuffer.get(payload); + + if (isMasked) { + for (int i = 0; i < payload.length; i++) { + payload[i] ^= maskKey[i & 3]; + } + } + + switch (opcode) { + case WebSocketOpcode.BINARY: + handler.onBinaryMessage(this, payload); + break; + case WebSocketOpcode.PING: + try { + writeFrame(WebSocketOpcode.PONG, payload, payload.length); + } catch (IOException e) { + LOG.error("Failed to send pong", e); + } + break; + case WebSocketOpcode.CLOSE: { + int code = WebSocketCloseCode.NORMAL_CLOSURE; + if (payload.length >= 2) { + code = ((payload[0] & 0xFF) << 8) | (payload[1] & 0xFF); + } + try { + sendClose(code, null); + } catch (IOException e) { + // client may have already disconnected + } + ClientHandler.this.running.set(false); + isClosed = true; + break; + } + } + } + + recvBuffer.compact(); + recvBuffer.flip(); + } + + private boolean performHandshake() throws IOException { + StringBuilder request = new StringBuilder(); + byte[] buf = new byte[1]; + while (true) { + int read = in.read(buf); + if (read <= 0) { + return false; + } + request.append((char) buf[0]); + if (request.toString().endsWith("\r\n\r\n")) { + break; + } + if (request.length() > 8192) { + return false; + } + } + + String key = null; + for (String line : request.toString().split("\r\n")) { + if (line.toLowerCase().startsWith("sec-websocket-key:")) { + key = line.substring(18).trim(); + break; + } + } + + if (key == null) { + return false; + } + + String acceptKey = computeAcceptKey(key); + + String response = "HTTP/1.1 101 Switching Protocols\r\n" + + "Upgrade: websocket\r\n" + + "Connection: Upgrade\r\n" + + "Sec-WebSocket-Accept: " + acceptKey + "\r\n" + + "\r\n"; + out.write(response.getBytes(StandardCharsets.US_ASCII)); + out.flush(); + + return true; + } + + private synchronized void writeFrame(int opcode, byte[] payload, int length) throws IOException { + // first byte: FIN + opcode + out.write(0x80 | (opcode & 0x0F)); + + // payload length (unmasked - server to client) + if (length <= 125) { + out.write(length); + } else if (length <= 65_535) { + out.write(126); + out.write((length >> 8) & 0xFF); + out.write(length & 0xFF); + } else { + out.write(127); + ByteBuffer lenBuf = ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN); + lenBuf.putLong(length); + out.write(lenBuf.array()); + } + + // payload + out.write(payload, 0, length); + out.flush(); + } + + void start() { + if (running.getAndSet(true)) { + return; + } + + readThread = new Thread(() -> { + try { + socket.setSoTimeout(100); + + in = socket.getInputStream(); + out = socket.getOutputStream(); + + if (!performHandshake()) { + LOG.error("Handshake failed"); + return; + } + + byte[] readBuf = new byte[8192]; + + while (running.get() && !isClosed) { + int read; + try { + read = in.read(readBuf); + } catch (SocketTimeoutException e) { + continue; + } + if (read <= 0) { + break; + } + + // append to recvBuffer + recvBuffer.compact(); + if (recvBuffer.remaining() < read) { + // should not happen with 64k buffer in tests + LOG.error("Receive buffer overflow"); + break; + } + recvBuffer.put(readBuf, 0, read); + recvBuffer.flip(); + + handleRead(); + } + } catch (IOException e) { + if (running.get()) { + LOG.error("Client error", e); + } + } + }, "WebSocket-Client-" + socket.getPort()); + readThread.start(); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/WebSocketFrameParserTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/WebSocketFrameParserTest.java new file mode 100644 index 0000000..5f78906 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/WebSocketFrameParserTest.java @@ -0,0 +1,736 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.websocket; + +import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; +import io.questdb.client.cutlass.qwp.websocket.WebSocketFrameParser; +import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; + +/** + * Tests for the client-side WebSocket frame parser. + * The client parser expects unmasked frames (from the server) + * and rejects masked frames. + */ +public class WebSocketFrameParserTest { + + @Test + public void testControlFrameBetweenFragments() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(64); + try { + WebSocketFrameParser parser = new WebSocketFrameParser(); + + // First data fragment + writeBytes(buf, (byte) 0x01, (byte) 0x02, (byte) 'H', (byte) 'i'); + parser.parse(buf, buf + 4); + Assert.assertFalse(parser.isFin()); + Assert.assertEquals(WebSocketOpcode.TEXT, parser.getOpcode()); + + // Ping in the middle (control frame, FIN must be 1) + parser.reset(); + writeBytes(buf, (byte) 0x89, (byte) 0x00); + parser.parse(buf, buf + 2); + Assert.assertTrue(parser.isFin()); + Assert.assertEquals(WebSocketOpcode.PING, parser.getOpcode()); + + // Final data fragment + parser.reset(); + writeBytes(buf, (byte) 0x80, (byte) 0x01, (byte) '!'); + parser.parse(buf, buf + 3); + Assert.assertTrue(parser.isFin()); + Assert.assertEquals(WebSocketOpcode.CONTINUATION, parser.getOpcode()); + } finally { + freeBuffer(buf, 64); + } + }); + } + + @Test + public void testOpcodeIsControlFrame() throws Exception { + assertMemoryLeak(() -> { + Assert.assertFalse(WebSocketOpcode.isControlFrame(WebSocketOpcode.CONTINUATION)); + Assert.assertFalse(WebSocketOpcode.isControlFrame(WebSocketOpcode.TEXT)); + Assert.assertFalse(WebSocketOpcode.isControlFrame(WebSocketOpcode.BINARY)); + Assert.assertTrue(WebSocketOpcode.isControlFrame(WebSocketOpcode.CLOSE)); + Assert.assertTrue(WebSocketOpcode.isControlFrame(WebSocketOpcode.PING)); + Assert.assertTrue(WebSocketOpcode.isControlFrame(WebSocketOpcode.PONG)); + }); + } + + @Test + public void testOpcodeIsDataFrame() throws Exception { + assertMemoryLeak(() -> { + Assert.assertTrue(WebSocketOpcode.isDataFrame(WebSocketOpcode.CONTINUATION)); + Assert.assertTrue(WebSocketOpcode.isDataFrame(WebSocketOpcode.TEXT)); + Assert.assertTrue(WebSocketOpcode.isDataFrame(WebSocketOpcode.BINARY)); + Assert.assertFalse(WebSocketOpcode.isDataFrame(WebSocketOpcode.CLOSE)); + Assert.assertFalse(WebSocketOpcode.isDataFrame(WebSocketOpcode.PING)); + Assert.assertFalse(WebSocketOpcode.isDataFrame(WebSocketOpcode.PONG)); + }); + } + + @Test + public void testOpcodeIsValid() throws Exception { + assertMemoryLeak(() -> { + Assert.assertTrue(WebSocketOpcode.isValid(WebSocketOpcode.CONTINUATION)); + Assert.assertTrue(WebSocketOpcode.isValid(WebSocketOpcode.TEXT)); + Assert.assertTrue(WebSocketOpcode.isValid(WebSocketOpcode.BINARY)); + Assert.assertFalse(WebSocketOpcode.isValid(3)); + Assert.assertFalse(WebSocketOpcode.isValid(4)); + Assert.assertFalse(WebSocketOpcode.isValid(5)); + Assert.assertFalse(WebSocketOpcode.isValid(6)); + Assert.assertFalse(WebSocketOpcode.isValid(7)); + Assert.assertTrue(WebSocketOpcode.isValid(WebSocketOpcode.CLOSE)); + Assert.assertTrue(WebSocketOpcode.isValid(WebSocketOpcode.PING)); + Assert.assertTrue(WebSocketOpcode.isValid(WebSocketOpcode.PONG)); + Assert.assertFalse(WebSocketOpcode.isValid(0xB)); + Assert.assertFalse(WebSocketOpcode.isValid(0xF)); + }); + } + + @Test + public void testParse16BitLength() throws Exception { + assertMemoryLeak(() -> { + int payloadLen = 1000; + long buf = allocateBuffer(payloadLen + 16); + try { + writeBytes(buf, + (byte) 0x82, // FIN + BINARY + (byte) 126, // 16-bit length follows + (byte) (payloadLen >> 8), // Length high byte + (byte) (payloadLen & 0xFF) // Length low byte + ); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 4 + payloadLen); + + Assert.assertEquals(4 + payloadLen, consumed); + Assert.assertEquals(payloadLen, parser.getPayloadLength()); + } finally { + freeBuffer(buf, payloadLen + 16); + } + }); + } + + @Test + public void testParse64BitLength() throws Exception { + assertMemoryLeak(() -> { + long payloadLen = 70_000L; + long buf = allocateBuffer((int) payloadLen + 16); + try { + Unsafe.getUnsafe().putByte(buf, (byte) 0x82); + Unsafe.getUnsafe().putByte(buf + 1, (byte) 127); + Unsafe.getUnsafe().putLong(buf + 2, Long.reverseBytes(payloadLen)); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 10 + payloadLen); + + Assert.assertEquals(10 + payloadLen, consumed); + Assert.assertEquals(payloadLen, parser.getPayloadLength()); + } finally { + freeBuffer(buf, (int) payloadLen + 16); + } + }); + } + + @Test + public void testParse7BitLength() throws Exception { + assertMemoryLeak(() -> { + for (int len = 0; len <= 125; len++) { + long buf = allocateBuffer(256); + try { + writeBytes(buf, (byte) 0x82, (byte) len); + for (int i = 0; i < len; i++) { + Unsafe.getUnsafe().putByte(buf + 2 + i, (byte) i); + } + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 2 + len); + + Assert.assertEquals(2 + len, consumed); + Assert.assertEquals(len, parser.getPayloadLength()); + } finally { + freeBuffer(buf, 256); + } + } + }); + } + + @Test + public void testParseBinaryFrame() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 0x00); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 2); + + Assert.assertEquals(WebSocketOpcode.BINARY, parser.getOpcode()); + } finally { + freeBuffer(buf, 16); + } + }); + } + + @Test + public void testParseCloseFrame() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, + (byte) 0x88, // FIN + CLOSE + (byte) 0x02, // Length 2 (just the code) + (byte) 0x03, (byte) 0xE8 // 1000 in big-endian + ); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 4); + + Assert.assertEquals(WebSocketOpcode.CLOSE, parser.getOpcode()); + Assert.assertEquals(2, parser.getPayloadLength()); + } finally { + freeBuffer(buf, 16); + } + }); + } + + @Test + public void testParseCloseFrameEmpty() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x88, (byte) 0x00); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 2); + + Assert.assertEquals(WebSocketOpcode.CLOSE, parser.getOpcode()); + Assert.assertEquals(0, parser.getPayloadLength()); + } finally { + freeBuffer(buf, 16); + } + }); + } + + @Test + public void testParseCloseFrameWithReason() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(64); + try { + String reason = "Normal closure"; + byte[] reasonBytes = reason.getBytes(StandardCharsets.UTF_8); + + Unsafe.getUnsafe().putByte(buf, (byte) 0x88); + Unsafe.getUnsafe().putByte(buf + 1, (byte) (2 + reasonBytes.length)); + Unsafe.getUnsafe().putShort(buf + 2, Short.reverseBytes((short) 1000)); + for (int i = 0; i < reasonBytes.length; i++) { + Unsafe.getUnsafe().putByte(buf + 4 + i, reasonBytes[i]); + } + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 4 + reasonBytes.length); + + Assert.assertEquals(WebSocketOpcode.CLOSE, parser.getOpcode()); + Assert.assertEquals(2 + reasonBytes.length, parser.getPayloadLength()); + } finally { + freeBuffer(buf, 64); + } + }); + } + + @Test + public void testParseContinuationFrame() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x00, (byte) 0x05, (byte) 0x01, (byte) 0x02, (byte) 0x03, (byte) 0x04, (byte) 0x05); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 7); + + Assert.assertEquals(WebSocketOpcode.CONTINUATION, parser.getOpcode()); + Assert.assertFalse(parser.isFin()); + } finally { + freeBuffer(buf, 16); + } + }); + } + + @Test + public void testParseEmptyBuffer() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf); + + Assert.assertEquals(0, consumed); + } finally { + freeBuffer(buf, 16); + } + }); + } + + @Test + public void testParseEmptyPayload() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 0x00); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 2); + + Assert.assertEquals(2, consumed); + Assert.assertEquals(0, parser.getPayloadLength()); + } finally { + freeBuffer(buf, 16); + } + }); + } + + @Test + public void testParseFragmentedMessage() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(64); + try { + // First fragment: opcode=TEXT, FIN=0 + writeBytes(buf, (byte) 0x01, (byte) 0x03, (byte) 'H', (byte) 'e', (byte) 'l'); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 5); + + Assert.assertEquals(5, consumed); + Assert.assertFalse(parser.isFin()); + Assert.assertEquals(WebSocketOpcode.TEXT, parser.getOpcode()); + + // Continuation: opcode=CONTINUATION, FIN=0 + parser.reset(); + writeBytes(buf, (byte) 0x00, (byte) 0x02, (byte) 'l', (byte) 'o'); + consumed = parser.parse(buf, buf + 4); + + Assert.assertEquals(4, consumed); + Assert.assertFalse(parser.isFin()); + Assert.assertEquals(WebSocketOpcode.CONTINUATION, parser.getOpcode()); + + // Final fragment: opcode=CONTINUATION, FIN=1 + parser.reset(); + writeBytes(buf, (byte) 0x80, (byte) 0x01, (byte) '!'); + consumed = parser.parse(buf, buf + 3); + + Assert.assertEquals(3, consumed); + Assert.assertTrue(parser.isFin()); + Assert.assertEquals(WebSocketOpcode.CONTINUATION, parser.getOpcode()); + } finally { + freeBuffer(buf, 64); + } + }); + } + + @Test + public void testParseIncompleteHeader16BitLength() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 126, (byte) 0x01); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 3); + + Assert.assertEquals(0, consumed); + Assert.assertEquals(WebSocketFrameParser.STATE_NEED_MORE, parser.getState()); + } finally { + freeBuffer(buf, 16); + } + }); + } + + @Test + public void testParseIncompleteHeader1Byte() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 1); + + Assert.assertEquals(0, consumed); + Assert.assertEquals(WebSocketFrameParser.STATE_NEED_MORE, parser.getState()); + } finally { + freeBuffer(buf, 16); + } + }); + } + + @Test + public void testParseIncompleteHeader64BitLength() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 127, (byte) 0, (byte) 0, (byte) 0, (byte) 0); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 6); + + Assert.assertEquals(0, consumed); + } finally { + freeBuffer(buf, 16); + } + }); + } + + @Test + public void testParseIncompletePayload() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 0x05, (byte) 0x01, (byte) 0x02); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 4); + + Assert.assertEquals(2, consumed); + Assert.assertEquals(5, parser.getPayloadLength()); + Assert.assertEquals(WebSocketFrameParser.STATE_NEED_PAYLOAD, parser.getState()); + } finally { + freeBuffer(buf, 16); + } + }); + } + + @Test + public void testParseMaxControlFrameSize() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(256); + try { + Unsafe.getUnsafe().putByte(buf, (byte) 0x89); // PING + Unsafe.getUnsafe().putByte(buf + 1, (byte) 125); + for (int i = 0; i < 125; i++) { + Unsafe.getUnsafe().putByte(buf + 2 + i, (byte) i); + } + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 127); + + Assert.assertEquals(127, consumed); + Assert.assertEquals(125, parser.getPayloadLength()); + Assert.assertNotEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + } finally { + freeBuffer(buf, 256); + } + }); + } + + @Test + public void testParseMinimalFrame() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 0x01, (byte) 0xFF); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 3); + + Assert.assertEquals(3, consumed); + Assert.assertTrue(parser.isFin()); + Assert.assertEquals(WebSocketOpcode.BINARY, parser.getOpcode()); + Assert.assertEquals(1, parser.getPayloadLength()); + } finally { + freeBuffer(buf, 16); + } + }); + } + + @Test + public void testParseMultipleFramesInBuffer() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(32); + try { + writeBytes(buf, + (byte) 0x82, (byte) 0x02, (byte) 0x01, (byte) 0x02, + (byte) 0x81, (byte) 0x03, (byte) 'a', (byte) 'b', (byte) 'c' + ); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + + int consumed = parser.parse(buf, buf + 9); + Assert.assertEquals(4, consumed); + Assert.assertEquals(WebSocketOpcode.BINARY, parser.getOpcode()); + Assert.assertEquals(2, parser.getPayloadLength()); + + parser.reset(); + consumed = parser.parse(buf + 4, buf + 9); + Assert.assertEquals(5, consumed); + Assert.assertEquals(WebSocketOpcode.TEXT, parser.getOpcode()); + Assert.assertEquals(3, parser.getPayloadLength()); + } finally { + freeBuffer(buf, 32); + } + }); + } + + @Test + public void testParsePingFrame() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x89, (byte) 0x04, (byte) 0x01, (byte) 0x02, (byte) 0x03, (byte) 0x04); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 6); + + Assert.assertEquals(WebSocketOpcode.PING, parser.getOpcode()); + Assert.assertEquals(4, parser.getPayloadLength()); + } finally { + freeBuffer(buf, 16); + } + }); + } + + @Test + public void testParsePongFrame() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x8A, (byte) 0x04, (byte) 0x01, (byte) 0x02, (byte) 0x03, (byte) 0x04); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 6); + + Assert.assertEquals(WebSocketOpcode.PONG, parser.getOpcode()); + } finally { + freeBuffer(buf, 16); + } + }); + } + + @Test + public void testParseTextFrame() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x81, (byte) 0x00); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 2); + + Assert.assertEquals(WebSocketOpcode.TEXT, parser.getOpcode()); + } finally { + freeBuffer(buf, 16); + } + }); + } + + @Test + public void testRejectCloseFrameWith1BytePayload() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x88, (byte) 0x01, (byte) 0x00); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 3); + + Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + Assert.assertEquals(WebSocketCloseCode.PROTOCOL_ERROR, parser.getErrorCode()); + } finally { + freeBuffer(buf, 16); + } + }); + } + + @Test + public void testRejectFragmentedControlFrame() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x09, (byte) 0x00); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 2); + + Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + Assert.assertEquals(WebSocketCloseCode.PROTOCOL_ERROR, parser.getErrorCode()); + } finally { + freeBuffer(buf, 16); + } + }); + } + + @Test + public void testRejectMaskedFrame() throws Exception { + assertMemoryLeak(() -> { + // Client-side parser rejects masked frames from the server + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 0x81, (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0xFF); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 7); + + Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + } finally { + freeBuffer(buf, 16); + } + }); + } + + @Test + public void testRejectOversizeControlFrame() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(256); + try { + writeBytes(buf, (byte) 0x89, (byte) 126, (byte) 0x00, (byte) 0x7E); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 4); + + Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + Assert.assertEquals(WebSocketCloseCode.PROTOCOL_ERROR, parser.getErrorCode()); + } finally { + freeBuffer(buf, 256); + } + }); + } + + @Test + public void testRejectRSV2Bit() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0xA2, (byte) 0x00); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 2); + + Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + } finally { + freeBuffer(buf, 16); + } + }); + } + + @Test + public void testRejectRSV3Bit() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x92, (byte) 0x00); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 2); + + Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + } finally { + freeBuffer(buf, 16); + } + }); + } + + @Test + public void testRejectReservedBits() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0xC2, (byte) 0x00); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 2); + + Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + Assert.assertEquals(WebSocketCloseCode.PROTOCOL_ERROR, parser.getErrorCode()); + } finally { + freeBuffer(buf, 16); + } + }); + } + + @Test + public void testRejectUnknownOpcode() throws Exception { + assertMemoryLeak(() -> { + for (int opcode : new int[]{3, 4, 5, 6, 7, 0xB, 0xC, 0xD, 0xE, 0xF}) { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) (0x80 | opcode), (byte) 0x00); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 2); + + Assert.assertEquals("Opcode " + opcode + " should be rejected", + WebSocketFrameParser.STATE_ERROR, parser.getState()); + } finally { + freeBuffer(buf, 16); + } + } + }); + } + + @Test + public void testReset() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 0x02, (byte) 0x01, (byte) 0x02); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 4); + + Assert.assertEquals(WebSocketOpcode.BINARY, parser.getOpcode()); + Assert.assertEquals(2, parser.getPayloadLength()); + + parser.reset(); + + Assert.assertEquals(0, parser.getOpcode()); + Assert.assertEquals(0, parser.getPayloadLength()); + Assert.assertEquals(WebSocketFrameParser.STATE_HEADER, parser.getState()); + } finally { + freeBuffer(buf, 16); + } + }); + } + + private static long allocateBuffer(int size) { + return Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); + } + + private static void freeBuffer(long address, int size) { + Unsafe.free(address, size, MemoryTag.NATIVE_DEFAULT); + } + + private static void writeBytes(long address, byte... bytes) { + for (int i = 0; i < bytes.length; i++) { + Unsafe.getUnsafe().putByte(address + i, bytes[i]); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/std/CharSequenceHashSetTest.java b/core/src/test/java/io/questdb/client/test/std/CharSequenceHashSetTest.java deleted file mode 100644 index 98a3d58..0000000 --- a/core/src/test/java/io/questdb/client/test/std/CharSequenceHashSetTest.java +++ /dev/null @@ -1,91 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test.std; - -import io.questdb.client.std.CharSequenceHashSet; -import io.questdb.client.std.Rnd; -import org.junit.Assert; -import org.junit.Test; - -import java.util.HashSet; - -public class CharSequenceHashSetTest { - - @Test - public void testNullHandling() { - Rnd rnd = new Rnd(); - CharSequenceHashSet set = new CharSequenceHashSet(); - int n = 1000; - - for (int i = 0; i < n; i++) { - set.add(next(rnd).toString()); - } - - Assert.assertFalse(set.contains(null)); - Assert.assertTrue(set.add((CharSequence) null)); - Assert.assertEquals(n + 1, set.size()); - Assert.assertFalse(set.add((CharSequence) null)); - Assert.assertEquals(n + 1, set.size()); - Assert.assertTrue(set.contains(null)); - } - - @Test - public void testStress() { - Rnd rnd = new Rnd(); - CharSequenceHashSet set = new CharSequenceHashSet(); - int n = 10000; - - for (int i = 0; i < n; i++) { - set.add(next(rnd).toString()); - } - - Assert.assertEquals(n, set.size()); - - HashSet check = new HashSet<>(); - for (int i = 0, m = set.size(); i < m; i++) { - check.add(set.get(i).toString()); - } - - Assert.assertEquals(n, check.size()); - - Rnd rnd2 = new Rnd(); - for (int i = 0; i < n; i++) { - Assert.assertTrue("at " + i, set.contains(next(rnd2))); - } - - Assert.assertEquals(n, set.size()); - - Rnd rnd3 = new Rnd(); - for (int i = 0; i < n; i++) { - Assert.assertFalse("at " + i, set.add(next(rnd3))); - } - - Assert.assertEquals(n, set.size()); - } - - private static CharSequence next(Rnd rnd) { - return rnd.nextChars((rnd.nextInt() & 15) + 10); - } -} \ No newline at end of file diff --git a/core/src/test/java/io/questdb/client/test/std/ConcurrentHashMapTest.java b/core/src/test/java/io/questdb/client/test/std/ConcurrentHashMapTest.java deleted file mode 100644 index 524fd64..0000000 --- a/core/src/test/java/io/questdb/client/test/std/ConcurrentHashMapTest.java +++ /dev/null @@ -1,156 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test.std; - -import io.questdb.client.std.ConcurrentHashMap; -import org.junit.Test; - -import java.util.Map; - -import static org.junit.Assert.*; - -public class ConcurrentHashMapTest { - - @Test - public void testCaseKey() { - ConcurrentHashMap map = new ConcurrentHashMap<>(4, false); - map.put("Table", "1"); - map.put("tAble", "2"); - map.put("TaBle", "3"); - map.put("TABle", "4"); - map.put("TaBLE", "5"); - map.putIfAbsent("TaBlE", "Hello"); - assertEquals(1, map.size()); - assertEquals(map.get("TABLE"), "5"); - assertEquals(((Map) map).get("TABLE"), "5"); - assertNull(((Map) map).get(42)); - - ConcurrentHashMap cs = new ConcurrentHashMap<>(5, 0.58F); - cs.put("Table", "1"); - cs.put("tAble", "2"); - cs.put("TaBle", "3"); - cs.put("TABle", "4"); - cs.put("TaBLE", "5"); - - ConcurrentHashMap ccs = new ConcurrentHashMap<>(cs); - assertEquals(ccs.size(), cs.size()); - assertEquals(ccs.get("TaBLE"), "5"); - assertNull(ccs.get("TABLE")); - - ConcurrentHashMap cci = new ConcurrentHashMap<>(cs, false); - assertEquals(1, cci.size()); - assertNotNull(cci.get("TaBLE")); - - ConcurrentHashMap ci = new ConcurrentHashMap<>(5, 0.58F, false); - ci.put("Table", "1"); - ci.put("tAble", "2"); - ci.put("TaBle", "3"); - ci.put("TABle", "4"); - ci.put("TaBLE", "5"); - assertEquals(1, ci.size()); - - ConcurrentHashMap.KeySetView ks0 = ConcurrentHashMap.newKeySet(4); - ks0.add("Table"); - ks0.add("tAble"); - ks0.add("TaBle"); - ks0.add("TABle"); - ks0.add("TaBLE"); - assertEquals(5, ks0.size()); - ConcurrentHashMap.KeySetView ks1 = ConcurrentHashMap.newKeySet(4, false); - ks1.add("Table"); - ks1.add("tAble"); - ks1.add("TaBle"); - ks1.add("TABle"); - ks1.add("TaBLE"); - assertEquals(1, ks1.size()); - } - - @Test - public void testCompute() { - ConcurrentHashMap map = identityMap(); - // add - assertEquals("X", map.compute("X", (k, v) -> "X")); - // ignore - map.compute("Y", (k, v) -> null); - assertFalse(map.containsKey("Y")); - // replace - assertEquals("X", map.compute("A", (k, v) -> "X")); - // remove - map.compute("B", (k, v) -> null); - assertFalse(map.containsKey("B")); - - try { - map.compute(null, (k, v) -> null); - fail("Null key"); - } catch (NullPointerException ignored) { - } - } - - @Test - public void testComputeIfAbsent() { - ConcurrentHashMap map = identityMap(); - - map.putIfAbsent("X", "X"); - assertTrue(map.containsKey("X")); - - assertEquals("A", map.computeIfAbsent("A", k -> "X")); - - map.computeIfAbsent("Y", k -> null); - assertFalse(map.containsKey("Y")); - - try { - map.computeIfAbsent(null, k -> null); - fail("Null key"); - } catch (NullPointerException ignored) { - } - } - - @Test - public void testComputeIfPresent() { - ConcurrentHashMap map = identityMap(); - - map.computeIfPresent("X", (k, v) -> "X"); - assertFalse(map.containsKey("X")); - - assertEquals("X", map.computeIfPresent("A", (k, v) -> "X")); - - try { - map.computeIfPresent(null, (k, v) -> null); - fail("Null key"); - } catch (NullPointerException ignored) { - } - } - - private static ConcurrentHashMap identityMap() { - ConcurrentHashMap identity = new ConcurrentHashMap<>(3); - assertTrue(identity.isEmpty()); - identity.put("A", "A"); - identity.put("B", "B"); - identity.put("C", "C"); - assertFalse(identity.isEmpty()); - assertEquals(3, identity.size()); - return identity; - } -} diff --git a/core/src/test/java/io/questdb/client/test/std/ConcurrentIntHashMapTest.java b/core/src/test/java/io/questdb/client/test/std/ConcurrentIntHashMapTest.java deleted file mode 100644 index f95b692..0000000 --- a/core/src/test/java/io/questdb/client/test/std/ConcurrentIntHashMapTest.java +++ /dev/null @@ -1,114 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test.std; - -import io.questdb.client.std.ConcurrentIntHashMap; -import org.junit.Assert; -import org.junit.Test; - -public class ConcurrentIntHashMapTest { - - @Test - public void testCompute() { - ConcurrentIntHashMap map = identityMap(); - // add - Assert.assertEquals(42, (long) map.compute(42, (k, v) -> 42)); - // ignore - map.compute(24, (k, v) -> null); - Assert.assertFalse(map.containsKey(24)); - // replace - Assert.assertEquals(42, (long) map.compute(1, (k, v) -> 42)); - // remove - map.compute(2, (k, v) -> null); - Assert.assertFalse(map.containsKey(2)); - } - - @Test - public void testComputeIfAbsent() { - ConcurrentIntHashMap map = identityMap(); - - map.putIfAbsent(42, 42); - Assert.assertTrue(map.containsKey(42)); - - Assert.assertEquals(1, (long) map.computeIfAbsent(1, k -> 42)); - - map.computeIfAbsent(142, k -> null); - Assert.assertFalse(map.containsKey(142)); - } - - @Test - public void testComputeIfPresent() { - ConcurrentIntHashMap map = identityMap(); - - map.computeIfPresent(42, (k, v) -> 42); - Assert.assertFalse(map.containsKey(42)); - - Assert.assertEquals(42, (long) map.computeIfPresent(1, (k, v) -> 42)); - } - - @Test - public void testNegativeKey() { - ConcurrentIntHashMap map = new ConcurrentIntHashMap<>(); - Assert.assertNull(map.get(-1)); - Assert.assertThrows(IllegalArgumentException.class, () -> map.put(-2, "a")); - Assert.assertThrows(IllegalArgumentException.class, () -> map.putIfAbsent(-3, "b")); - Assert.assertThrows(IllegalArgumentException.class, () -> map.compute(-4, (val1, val2) -> val2)); - Assert.assertThrows(IllegalArgumentException.class, () -> map.computeIfAbsent(-5, (val) -> "c")); - Assert.assertThrows(IllegalArgumentException.class, () -> map.computeIfPresent(-5, (val1, val2) -> val2)); - } - - @Test - public void testSmoke() { - ConcurrentIntHashMap map = new ConcurrentIntHashMap<>(4); - map.put(1, "1"); - map.put(2, "2"); - map.put(3, "3"); - map.put(4, "4"); - map.put(5, "5"); - map.putIfAbsent(5, "Hello"); - Assert.assertEquals(5, map.size()); - Assert.assertEquals(map.get(5), "5"); - Assert.assertNull(map.get(42)); - - ConcurrentIntHashMap.KeySetView ks = ConcurrentIntHashMap.newKeySet(4); - ks.add(1); - ks.add(2); - ks.add(3); - ks.add(4); - ks.add(5); - Assert.assertEquals(5, ks.size()); - } - - private static ConcurrentIntHashMap identityMap() { - ConcurrentIntHashMap identity = new ConcurrentIntHashMap<>(3, 0.9f); - Assert.assertTrue(identity.isEmpty()); - identity.put(1, 1); - identity.put(2, 2); - identity.put(3, 3); - Assert.assertFalse(identity.isEmpty()); - Assert.assertEquals(3, identity.size()); - return identity; - } -} diff --git a/core/src/test/java/io/questdb/client/test/std/NumbersTest.java b/core/src/test/java/io/questdb/client/test/std/NumbersTest.java index 03ac886..404d21b 100644 --- a/core/src/test/java/io/questdb/client/test/std/NumbersTest.java +++ b/core/src/test/java/io/questdb/client/test/std/NumbersTest.java @@ -41,37 +41,6 @@ public class NumbersTest { private final StringSink sink = new StringSink(); private Rnd rnd; - @Test - public void appendHexPadded() { - sink.clear(); - Numbers.appendHexPadded(sink, 0xff - 1, 2); - TestUtils.assertEquals("00fe", sink); - - sink.clear(); - Numbers.appendHexPadded(sink, 0xff0, 4); - TestUtils.assertEquals("00000ff0", sink); - - sink.clear(); - Numbers.appendHexPadded(sink, 1, 4); - TestUtils.assertEquals("00000001", sink); - - sink.clear(); - Numbers.appendHexPadded(sink, 0xff - 1, 3); - TestUtils.assertEquals("0000fe", sink); - - sink.clear(); - Numbers.appendHexPadded(sink, 0xff - 1, 1); - TestUtils.assertEquals("fe", sink); - - sink.clear(); - Numbers.appendHexPadded(sink, 0xffff, 0); - TestUtils.assertEquals("ffff", sink); - - sink.clear(); - Numbers.appendHexPadded(sink, 0, 8); - TestUtils.assertEquals("0000000000000000", sink); - } - @Test(expected = NumericException.class) public void parseExplicitDouble2() { Numbers.parseDouble("1234dx"); @@ -93,13 +62,6 @@ public void setUp() { sink.clear(); } - @Test - public void testAppendZeroLong256() { - sink.clear(); - Numbers.appendLong256(0, 0, 0, 0, sink); - TestUtils.assertEquals("0x00", sink); - } - @Test public void testCeilPow2() { assertEquals(16, ceilPow2(15)); @@ -112,11 +74,6 @@ public void testEmptyDouble() { Numbers.parseDouble("D"); } - @Test(expected = NumericException.class) - public void testEmptyFloat() { - Numbers.parseFloat("f"); - } - @Test(expected = NumericException.class) public void testEmptyLong() { Numbers.parseLong("L"); @@ -317,11 +274,6 @@ public void testHexInt() { public void testIntEdge() { Numbers.append(sink, Integer.MAX_VALUE); assertEquals(Integer.MAX_VALUE, Numbers.parseInt(sink)); - - sink.clear(); - - Numbers.append(sink, Integer.MIN_VALUE); - assertEquals(Integer.MIN_VALUE, Numbers.parseIntQuiet(sink)); } @Test @@ -354,40 +306,6 @@ public void testLongToString() { TestUtils.assertEquals("6103390276", sink); } - @Test(expected = NumericException.class) - public void testParse000Greedy0() throws NumericException { - Numbers.parseInt000Greedy("", 0, 0); - } - - @Test - public void testParse000Greedy1() throws NumericException { - String input = "2"; - long val = Numbers.parseInt000Greedy(input, 0, input.length()); - assertEquals(input.length(), Numbers.decodeHighInt(val)); - assertEquals(200, Numbers.decodeLowInt(val)); - } - - @Test - public void testParse000Greedy2() throws NumericException { - String input = "06"; - long val = Numbers.parseInt000Greedy(input, 0, input.length()); - assertEquals(input.length(), Numbers.decodeHighInt(val)); - assertEquals(60, Numbers.decodeLowInt(val)); - } - - @Test - public void testParse000Greedy3() throws NumericException { - String input = "219"; - long val = Numbers.parseInt000Greedy(input, 0, input.length()); - assertEquals(input.length(), Numbers.decodeHighInt(val)); - assertEquals(219, Numbers.decodeLowInt(val)); - } - - @Test(expected = NumericException.class) - public void testParse000Greedy4() throws NumericException { - Numbers.parseInt000Greedy("1234", 0, 4); - } - @Test public void testParseDouble() { @@ -496,123 +414,6 @@ public void testParseExplicitDouble() { assertEquals(1234.123d, Numbers.parseDouble("1234.123d"), 0.000001); } - @Test - public void testParseExplicitFloat() { - assertEquals(12345.02f, Numbers.parseFloat("12345.02f"), 0.0001f); - } - - @Test(expected = NumericException.class) - public void testParseExplicitFloat2() { - Numbers.parseFloat("12345.02fx"); - } - - @Test - public void testParseFloat() { - String s1 = "0.45677899234"; - assertEquals(Float.parseFloat(s1), Numbers.parseFloat(s1), 0.000000001); - - String s2 = "1.459983E35"; - assertEquals(Float.parseFloat(s2) / 1e35d, Numbers.parseFloat(s2) / 1e35d, 0.00001); - - String s3 = "0.000000023E-30"; - assertEquals(Float.parseFloat(s3), Numbers.parseFloat(s3), 0.000000001); - - // overflow - try { - Numbers.parseFloat("1.0000E-204"); - Assert.fail(); - } catch (NumericException ignored) { - } - - try { - Numbers.parseFloat("1E39"); - Assert.fail(); - } catch (NumericException ignored) { - } - - try { - Numbers.parseFloat("1.0E39"); - Assert.fail(); - } catch (NumericException ignored) { - } - - String s6 = "200E2"; - assertEquals(Float.parseFloat(s6), Numbers.parseFloat(s6), 0.000000001); - - String s7 = "NaN"; - assertEquals(Float.parseFloat(s7), Numbers.parseFloat(s7), 0.000000001); - - String s8 = "-Infinity"; - assertEquals(Float.parseFloat(s8), Numbers.parseFloat(s8), 0.000000001); - - // min exponent float - String s9 = "1.4e-45"; - assertEquals(1.4e-45f, Numbers.parseFloat(s9), 0.001); - - // false overflow - String s10 = "0003000.0e-46"; - assertEquals(1.4e-45f, Numbers.parseFloat(s10), 0.001); - - // false overflow - String s11 = "0.00001e40"; - assertEquals(1e35f, Numbers.parseFloat(s11), 0.001); - } - - @Test - public void testParseFloatCloseToZero() { - String s1 = "0.123456789"; - assertEquals(Float.parseFloat(s1), Numbers.parseFloat(s1), 0.000000001); - - String s2 = "0.12345678901234567890123456789E12"; - assertEquals(Float.parseFloat(s2), Numbers.parseFloat(s2), 0.000000001); - } - - @Test - public void testParseFloatIntegerLargerThanLongMaxValue() { - String s1 = "9223372036854775808"; - assertEquals(Float.parseFloat(s1), Numbers.parseFloat(s1), 0.000000001); - - String s2 = "9223372036854775808123"; - assertEquals(Float.parseFloat(s2), Numbers.parseFloat(s2), 0.000000001); - - String s3 = "9223372036854775808123922337203685477"; - assertEquals(Float.parseFloat(s3), Numbers.parseFloat(s3), 0.000000001); - - String s4 = "92233720368547758081239223372036854771"; - assertEquals(Float.parseFloat(s4), Numbers.parseFloat(s4), 0.000000001); - } - - @Test - public void testParseFloatLargerThanLongMaxValue() throws NumericException { - String s1 = "9223372036854775808.0123456789"; - assertEquals(Float.parseFloat(s1), Numbers.parseFloat(s1), 0.000000001); - - String s2 = "9223372036854775808.0123456789"; - assertEquals(Float.parseFloat(s2), Numbers.parseFloat(s2), 0.000000001); - - String s3 = "9223372036854775808123.0123456789"; - assertEquals(Float.parseFloat(s3), Numbers.parseFloat(s3), 0.000000001); - - String s4 = "922337203685477580812392233720368547758081.01239223372036854775808123"; // overflow - try { - Numbers.parseFloat(s4); - Assert.fail(); - } catch (NumericException ignored) { - } - } - - @Test - public void testParseFloatNegativeZero() throws NumericException { - float actual = Numbers.parseFloat("-0.0"); - - //check it's zero at all - assertEquals(0, actual, 0.0); - - //check it's *negative* zero - float res = 1 / actual; - assertEquals(Float.NEGATIVE_INFINITY, res, 0.0); - } - @Test public void testParseIPv4() { assertEquals(84413540, Numbers.parseIPv4("5.8.12.100")); @@ -677,12 +478,6 @@ public void testParseIPv4Overflow3() { Numbers.parseIPv4("12.1.3500.2"); } - @Test - public void testParseIPv4Quiet() { - assertEquals(0, Numbers.parseIPv4Quiet(null)); - assertEquals(0, Numbers.parseIPv4Quiet("NaN")); - } - @Test(expected = NumericException.class) public void testParseIPv4SignOnly() { Numbers.parseIPv4("-"); @@ -793,28 +588,6 @@ public void testParseIntSignOnly() { Numbers.parseInt("-"); } - @Test - public void testParseIntToDelim() { - String in = "1234x5"; - long val = Numbers.parseIntSafely(in, 0, in.length()); - assertEquals(1234, Numbers.decodeLowInt(val)); - assertEquals(4, Numbers.decodeHighInt(val)); - } - - @Test(expected = NumericException.class) - public void testParseIntToDelimEmpty() { - String in = "x"; - Numbers.parseIntSafely(in, 0, in.length()); - } - - @Test - public void testParseIntToDelimNoChar() { - String in = "12345"; - long val = Numbers.parseIntSafely(in, 0, in.length()); - assertEquals(12345, Numbers.decodeLowInt(val)); - assertEquals(5, Numbers.decodeHighInt(val)); - } - @Test(expected = NumericException.class) public void testParseIntWrongChars() { Numbers.parseInt("123ab"); diff --git a/core/src/test/java/io/questdb/client/test/std/SecureRndTest.java b/core/src/test/java/io/questdb/client/test/std/SecureRndTest.java new file mode 100644 index 0000000..4dac697 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/std/SecureRndTest.java @@ -0,0 +1,120 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.std; + +import io.questdb.client.std.SecureRnd; +import org.junit.Assert; +import org.junit.Test; + +public class SecureRndTest { + + @Test + public void testConsecutiveCallsProduceDifferentValues() { + SecureRnd rnd = new SecureRnd(); + int prev = rnd.nextInt(); + boolean foundDifferent = false; + for (int i = 0; i < 100; i++) { + int next = rnd.nextInt(); + if (next != prev) { + foundDifferent = true; + break; + } + prev = next; + } + Assert.assertTrue("Expected different values from consecutive calls", foundDifferent); + } + + @Test + public void testDifferentInstancesProduceDifferentSequences() { + SecureRnd rnd1 = new SecureRnd(); + SecureRnd rnd2 = new SecureRnd(); + boolean foundDifferent = false; + for (int i = 0; i < 16; i++) { + if (rnd1.nextInt() != rnd2.nextInt()) { + foundDifferent = true; + break; + } + } + Assert.assertTrue("Two SecureRnd instances should produce different sequences", foundDifferent); + } + + @Test + public void testMultipleBlocksDoNotRepeat() { + SecureRnd rnd = new SecureRnd(); + // Consume more than one block (16 ints) to trigger block counter increment + int[] first16 = new int[16]; + for (int i = 0; i < 16; i++) { + first16[i] = rnd.nextInt(); + } + // Next 16 should be from a different block + boolean foundDifferent = false; + for (int i = 0; i < 16; i++) { + if (rnd.nextInt() != first16[i]) { + foundDifferent = true; + break; + } + } + Assert.assertTrue("Second block should differ from first", foundDifferent); + } + + // RFC 7539 Section 2.3.2 known-answer test + @Test + public void testRfc7539Section232TestVector() { + // Key: 00:01:02:03:...:1f + byte[] key = new byte[32]; + for (int i = 0; i < 32; i++) { + key[i] = (byte) i; + } + + // Nonce: 00:00:00:09:00:00:00:4a:00:00:00:00 + byte[] nonce = { + 0x00, 0x00, 0x00, 0x09, + 0x00, 0x00, 0x00, 0x4a, + 0x00, 0x00, 0x00, 0x00 + }; + + // Block counter = 1 + SecureRnd rnd = new SecureRnd(key, nonce, 1); + + // Expected output words (ChaCha state after adding original input) + // from RFC 7539 Section 2.3.2 + int[] expected = { + 0xe4e7f110, 0x15593bd1, 0x1fdd0f50, (int) 0xc47120a3, + (int) 0xc7f4d1c7, 0x0368c033, (int) 0x9aaa2204, 0x4e6cd4c3, + 0x466482d2, 0x09aa9f07, 0x05d7c214, (int) 0xa2028bd9, + (int) 0xd19c12b5, (int) 0xb94e16de, (int) 0xe883d0cb, 0x4e3c50a2, + }; + + for (int i = 0; i < 16; i++) { + int actual = rnd.nextInt(); + Assert.assertEquals( + "Mismatch at word " + i + ": expected 0x" + Integer.toHexString(expected[i]) + + " but got 0x" + Integer.toHexString(actual), + expected[i], + actual + ); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/std/VectFuzzTest.java b/core/src/test/java/io/questdb/client/test/std/VectFuzzTest.java index d7a7863..4d60684 100644 --- a/core/src/test/java/io/questdb/client/test/std/VectFuzzTest.java +++ b/core/src/test/java/io/questdb/client/test/std/VectFuzzTest.java @@ -29,14 +29,14 @@ import io.questdb.client.std.Os; import io.questdb.client.std.Unsafe; import io.questdb.client.std.Vect; -import io.questdb.client.test.tools.TestUtils; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Assert; import org.junit.Test; public class VectFuzzTest { @Test public void testMemmove() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { int maxSize = 1024 * 1024; int[] sizes = {1024, 4096, maxSize}; int buffSize = 1024 + 4096 + maxSize; diff --git a/core/src/test/java/io/questdb/client/test/std/str/Utf8sTest.java b/core/src/test/java/io/questdb/client/test/std/str/Utf8sTest.java index 1b4fae8..b74bde9 100644 --- a/core/src/test/java/io/questdb/client/test/std/str/Utf8sTest.java +++ b/core/src/test/java/io/questdb/client/test/std/str/Utf8sTest.java @@ -272,15 +272,6 @@ public void testUtf8Support() { } } - @Test - public void testValidateUtf8() { - Assert.assertEquals(0, Utf8s.validateUtf8(Utf8String.EMPTY)); - Assert.assertEquals(3, Utf8s.validateUtf8(utf8("abc"))); - Assert.assertEquals(10, Utf8s.validateUtf8(utf8("привет мир"))); - // invalid UTF-8 - Assert.assertEquals(-1, Utf8s.validateUtf8(new Utf8String(new byte[]{(byte) 0x80}, false))); - } - private static byte b(int n) { return (byte) n; } diff --git a/core/src/test/java/io/questdb/client/test/tools/TestUtils.java b/core/src/test/java/io/questdb/client/test/tools/TestUtils.java index e380993..9c9ed99 100644 --- a/core/src/test/java/io/questdb/client/test/tools/TestUtils.java +++ b/core/src/test/java/io/questdb/client/test/tools/TestUtils.java @@ -25,12 +25,10 @@ package io.questdb.client.test.tools; import io.questdb.client.std.BinarySequence; -import io.questdb.client.std.Chars; import io.questdb.client.std.Files; import io.questdb.client.std.IntList; import io.questdb.client.std.LongList; import io.questdb.client.std.MemoryTag; -import io.questdb.client.std.Numbers; import io.questdb.client.std.ObjList; import io.questdb.client.std.Os; import io.questdb.client.std.QuietCloseable; @@ -86,7 +84,7 @@ public static void assertContains(String message, CharSequence sequence, CharSeq if (term.length() == 0) { return; } - if (Chars.contains(sequence, term)) { + if (sequence.toString().contains(term.toString())) { return; } Assert.fail((message != null ? message + ": '" : "'") + sequence + "' does not contain: " + term); @@ -351,7 +349,7 @@ public static void assertNotContains(String message, CharSequence sequence, Char Assert.fail(formatted + "Cannot assert that sequence does not contain an empty term; an empty term is always considered contained by definition."); } - if (!Chars.contains(sequence, term)) { + if (!sequence.toString().contains(term.toString())) { return; } @@ -448,9 +446,7 @@ public static String getTestResourcePath(String resourceName) { } public static String ipv4ToString(int ip) { - StringSink sink = getTlSink(); - Numbers.intToIPv4Sink(sink, ip); - return sink.toString(); + return ((ip >> 24) & 0xff) + "." + ((ip >> 16) & 0xff) + "." + ((ip >> 8) & 0xff) + "." + (ip & 0xff); } public static String readStringFromFile(File file) { diff --git a/core/src/test/java/module-info.java b/core/src/test/java/module-info.java index 0f649b5..a398b59 100644 --- a/core/src/test/java/module-info.java +++ b/core/src/test/java/module-info.java @@ -35,4 +35,6 @@ exports io.questdb.client.test; exports io.questdb.client.test.cairo; + exports io.questdb.client.test.cutlass.line; + exports io.questdb.client.test.cutlass.qwp.client; }