diff --git a/flatbuffers/rowBatch.fbs b/flatbuffers/rowBatch.fbs index 80d173250f..26fc80af9f 100644 --- a/flatbuffers/rowBatch.fbs +++ b/flatbuffers/rowBatch.fbs @@ -93,6 +93,11 @@ table IntColumnVectorFlat { vector : [int]; } +table ShortColumnVectorFlat { + base : ColumnVectorBaseFlat; + vector : [short]; +} + table LongColumnVectorFlat { base : ColumnVectorBaseFlat; vector : [long]; @@ -138,7 +143,8 @@ union ColumnVectorFlat { LongDecimalColumnVectorFlat, TimeColumnVectorFlat, TimestampColumnVectorFlat, - VectorColumnVectorFlat + VectorColumnVectorFlat, + ShortColumnVectorFlat } table VectorizedRowBatchFlat { diff --git a/pixels-amphi/src/main/java/io/pixelsdb/pixels/amphi/downloader/PeerDownloader.java b/pixels-amphi/src/main/java/io/pixelsdb/pixels/amphi/downloader/PeerDownloader.java index 3fe50464c8..8d1b505216 100644 --- a/pixels-amphi/src/main/java/io/pixelsdb/pixels/amphi/downloader/PeerDownloader.java +++ b/pixels-amphi/src/main/java/io/pixelsdb/pixels/amphi/downloader/PeerDownloader.java @@ -361,7 +361,13 @@ private static GenericRecord castToGenericRecord(Schema schema, List col record.put(columns.get(i).getName(), bcv.vector[rowIdx]); break; case SHORT: + ShortColumnVector scv = (ShortColumnVector) columnVectors.get(i); + record.put(columns.get(i).getName(), (int) scv.vector[rowIdx]); + break; case INT: + IntColumnVector icv = (IntColumnVector) columnVectors.get(i); + record.put(columns.get(i).getName(), icv.vector[rowIdx]); + break; case LONG: LongColumnVector lcv = (LongColumnVector) columnVectors.get(i); record.put(columns.get(i).getName(), lcv.vector[rowIdx]); diff --git a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/IndexedPixelsConsumer.java b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/IndexedPixelsConsumer.java index 5c7c2b3ad7..1cfa8299fa 100644 --- a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/IndexedPixelsConsumer.java +++ b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/IndexedPixelsConsumer.java @@ -308,7 +308,7 @@ public PerVirtualNodeWriter(PixelsWriter writer, File file, Path path, NodeProto this.prevRgId = this.rgId; this.rgRowOffset = 0; this.rowCounter = 0; - this.rowBatch = schema.createRowBatchWithHiddenColumn(pixelStride, TypeDescription.Mode.NONE); + this.rowBatch = schema.createRowBatchWithHiddenColumn(pixelStride); this.vNodeId = vNodeId; this.indexService = indexServices.computeIfAbsent(node.getAddress(), nodeInfo -> RPCIndexService.CreateInstance(nodeInfo, indexServerPort)); diff --git a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/SimplePixelsConsumer.java b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/SimplePixelsConsumer.java index c2e360ed45..134720bed1 100644 --- a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/SimplePixelsConsumer.java +++ b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/SimplePixelsConsumer.java @@ -57,7 +57,7 @@ public SimplePixelsConsumer(BlockingQueue queue, Parameters parameters, ConcurrentLinkedQueue loadedInfos) { super(queue, parameters, loadedInfos); - this.rowBatch = schema.createRowBatchWithHiddenColumn(pixelStride, TypeDescription.Mode.NONE); + this.rowBatch = schema.createRowBatchWithHiddenColumn(pixelStride); } @Override diff --git a/pixels-core/src/main/java/io/pixelsdb/pixels/core/TypeDescription.java b/pixels-core/src/main/java/io/pixelsdb/pixels/core/TypeDescription.java index d1a3d17733..2656a388ae 100644 --- a/pixels-core/src/main/java/io/pixelsdb/pixels/core/TypeDescription.java +++ b/pixels-core/src/main/java/io/pixelsdb/pixels/core/TypeDescription.java @@ -199,8 +199,8 @@ public enum Category */ BOOLEAN(true, boolean.class, byte.class, "boolean"), BYTE(true, byte.class, byte.class, "tinyint", "byte"), - SHORT(true, short.class, long.class, "smallint", "short"), - INT(true, int.class, long.class, "integer", "int"), + SHORT(true, short.class, short.class, "smallint", "short"), + INT(true, int.class, int.class, "integer", "int"), LONG(true, long.class, long.class, "bigint", "long"), FLOAT(true, float.class, long.class, "float", "real"), DOUBLE(true, double.class, long.class, "double"), @@ -1222,7 +1222,7 @@ public int getMaximumId() return maxId; } - private ColumnVector createColumn(int maxSize, int mode, boolean... useEncodedVector) + private ColumnVector createColumn(int maxSize, int vectorLayout, boolean... useEncodedVector) { requireNonNull(useEncodedVector, "columnsEncoded should not be null"); // the length of useEncodedVector is already checked, not need to check again. @@ -1232,15 +1232,17 @@ private ColumnVector createColumn(int maxSize, int mode, boolean... useEncodedVe case BYTE: return new ByteColumnVector(maxSize); case SHORT: - case INT: - if (Mode.match(mode, Mode.CREATE_INT_VECTOR_FOR_INT)) + if (VectorLayout.match(vectorLayout, VectorLayout.SHORT_AS_LONG)) { - return new IntColumnVector(maxSize); + return new LongColumnVector(maxSize); } - else + return new ShortColumnVector(maxSize); + case INT: + if (VectorLayout.match(vectorLayout, VectorLayout.INT_AS_LONG)) { return new LongColumnVector(maxSize); } + return new IntColumnVector(maxSize); case LONG: return new LongColumnVector(maxSize); case DATE: @@ -1276,7 +1278,7 @@ private ColumnVector createColumn(int maxSize, int mode, boolean... useEncodedVe ColumnVector[] fieldVector = new ColumnVector[children.size()]; for (int i = 0; i < fieldVector.length; ++i) { - fieldVector[i] = children.get(i).createColumn(maxSize, mode, useEncodedVector[i]); + fieldVector[i] = children.get(i).createColumn(maxSize, vectorLayout, useEncodedVector[i]); } return new StructColumnVector(maxSize, fieldVector); } @@ -1287,7 +1289,7 @@ private ColumnVector createColumn(int maxSize, int mode, boolean... useEncodedVe } } - public VectorizedRowBatch createRowBatch(int maxSize, int mode, boolean... useEncodedVector) + public VectorizedRowBatch createRowBatch(int maxSize, int vectorLayout, boolean... useEncodedVector) { VectorizedRowBatch result; if (category == Category.STRUCT) @@ -1299,7 +1301,7 @@ public VectorizedRowBatch createRowBatch(int maxSize, int mode, boolean... useEn for (int i = 0; i < result.cols.length; ++i) { String fieldName = fieldNames.get(i); - ColumnVector cv = children.get(i).createColumn(maxSize, mode, + ColumnVector cv = children.get(i).createColumn(maxSize, vectorLayout, useEncodedVector.length != 0 && useEncodedVector[i]); int originId = columnNames.indexOf(fieldName); if (originId >= 0) @@ -1319,24 +1321,24 @@ public VectorizedRowBatch createRowBatch(int maxSize, int mode, boolean... useEn checkArgument(useEncodedVector.length == 0 || useEncodedVector.length == 1, "for null structure type, there can be only 0 or 1 elements in useEncodedVector"); result = new VectorizedRowBatch(1, maxSize); - result.cols[0] = createColumn(maxSize, mode, + result.cols[0] = createColumn(maxSize, vectorLayout, useEncodedVector.length == 1 && useEncodedVector[0]); } result.reset(); return result; } - public VectorizedRowBatch createRowBatch() + public VectorizedRowBatch createRowBatch(int maxSize, boolean... useEncodedVector) { - return createRowBatch(VectorizedRowBatch.DEFAULT_SIZE, Mode.NONE); + return createRowBatch(maxSize, VectorLayout.NONE, useEncodedVector); } - public VectorizedRowBatch createRowBatch(int size) + public VectorizedRowBatch createRowBatch() { - return createRowBatch(size, Mode.NONE); + return createRowBatch(VectorizedRowBatch.DEFAULT_SIZE, VectorLayout.NONE); } - public VectorizedRowBatch createRowBatchWithHiddenColumn(int maxSize, int mode, boolean... useEncodedVector) + public VectorizedRowBatch createRowBatchWithHiddenColumn(int maxSize, int vectorLayout, boolean... useEncodedVector) { VectorizedRowBatch result; if (category == Category.STRUCT) @@ -1349,7 +1351,7 @@ public VectorizedRowBatch createRowBatchWithHiddenColumn(int maxSize, int mode, for (int i = 0; i < result.cols.length - 1; ++i) { String fieldName = fieldNames.get(i); - ColumnVector cv = children.get(i).createColumn(maxSize, mode, + ColumnVector cv = children.get(i).createColumn(maxSize, vectorLayout, useEncodedVector.length != 0 && useEncodedVector[i]); int originId = columnNames.indexOf(fieldName); if (originId >= 0) @@ -1371,7 +1373,7 @@ public VectorizedRowBatch createRowBatchWithHiddenColumn(int maxSize, int mode, checkArgument(useEncodedVector.length == 0 || useEncodedVector.length == 1, "for null structure type, there can be only 0 or 1 elements in useEncodedVector"); result = new VectorizedRowBatch(2, maxSize); - result.cols[0] = createColumn(maxSize, mode, + result.cols[0] = createColumn(maxSize, vectorLayout, useEncodedVector.length == 1 && useEncodedVector[0]); result.cols[1] = new LongColumnVector(maxSize); } @@ -1379,9 +1381,43 @@ public VectorizedRowBatch createRowBatchWithHiddenColumn(int maxSize, int mode, return result; } + public VectorizedRowBatch createRowBatchWithHiddenColumn(int maxSize, boolean... useEncodedVector) + { + return createRowBatchWithHiddenColumn(maxSize, VectorLayout.NONE, useEncodedVector); + } + public VectorizedRowBatch createRowBatchWithHiddenColumn() { - return createRowBatchWithHiddenColumn(VectorizedRowBatch.DEFAULT_SIZE, Mode.NONE); + return createRowBatchWithHiddenColumn(VectorizedRowBatch.DEFAULT_SIZE, VectorLayout.NONE); + } + + /** + * The column vector layouts used when creating column vectors and row batches. + * These flags control whether short/int columns are widened into + * {@link io.pixelsdb.pixels.core.vector.LongColumnVector} for backward + * compatibility with old query engines (e.g. Trino 405, Presto 0.279), + * or created as their native dedicated vectors + * (e.g. {@link io.pixelsdb.pixels.core.vector.ShortColumnVector}, + * {@link io.pixelsdb.pixels.core.vector.IntColumnVector}). + */ + public static final class VectorLayout + { + public static final int NONE = 0; + /** + * Read SHORT columns as {@link io.pixelsdb.pixels.core.vector.LongColumnVector} + * instead of {@link io.pixelsdb.pixels.core.vector.ShortColumnVector}. + */ + public static final int SHORT_AS_LONG = 0x01; + /** + * Read INT columns as {@link io.pixelsdb.pixels.core.vector.LongColumnVector} + * instead of {@link io.pixelsdb.pixels.core.vector.IntColumnVector}. + */ + public static final int INT_AS_LONG = 0x02; + + public static boolean match(int layout1, int layout2) + { + return (layout1 & layout2) != 0; + } } /** @@ -1674,23 +1710,6 @@ public TypeDescription findSubtype(int goal) } } - /** - * The type related modes used when creating column vectors and row batches. - */ - public static final class Mode - { - public static final int NONE = 0; - /** - * Create {@link IntColumnVector} for INT type. - */ - public static final int CREATE_INT_VECTOR_FOR_INT = 0x01; - - public static boolean match(int mode1, int mode2) - { - return (mode1 & mode2) != 0; - } - } - /** * Serializes one cell from a {@link ColumnVector} at the given row index into * the canonical byte format, using the same encoding as {@link #convertSqlStringToByte}. @@ -1708,19 +1727,26 @@ public byte[] convertColumnVectorToByte(ColumnVector col, int row) case BYTE: return new byte[]{((ByteColumnVector) col).vector[row]}; case SHORT: + { + short shortValue = col instanceof ShortColumnVector ? + ((ShortColumnVector) col).vector[row] : + (short) ((LongColumnVector) col).vector[row]; + return ByteBuffer.allocate(Short.BYTES).putShort(shortValue).array(); + } case INT: { - int value = col instanceof IntColumnVector ? + int intValue = col instanceof IntColumnVector ? ((IntColumnVector) col).vector[row] : (int) ((LongColumnVector) col).vector[row]; - return ByteBuffer.allocate(Integer.BYTES).putInt(value).array(); + return ByteBuffer.allocate(Integer.BYTES).putInt(intValue).array(); } case LONG: return ByteBuffer.allocate(Long.BYTES).putLong(((LongColumnVector) col).vector[row]).array(); case DATE: return ByteBuffer.allocate(Integer.BYTES).putInt(((DateColumnVector) col).dates[row]).array(); case TIME: - return ByteBuffer.allocate(Integer.BYTES).putInt(((TimeColumnVector) col).times[row]).array(); + return ByteBuffer.allocate(Integer.BYTES) + .putInt(((TimeColumnVector) col).times[row]).array(); case TIMESTAMP: return ByteBuffer.allocate(Long.BYTES).putLong(((TimestampColumnVector) col).times[row]).array(); case FLOAT: @@ -1781,6 +1807,11 @@ public byte[] convertSqlStringToByte(String value) return new byte[]{parsedByte}; } case SHORT: + { + short shortValue = Short.parseShort(value); + bytes = ByteBuffer.allocate(Short.BYTES).putShort(shortValue).array(); + break; + } case INT: { int intValue = Integer.parseInt(value); diff --git a/pixels-core/src/main/java/io/pixelsdb/pixels/core/encoding/Encoder.java b/pixels-core/src/main/java/io/pixelsdb/pixels/core/encoding/Encoder.java index 2f5fa243e3..7a1e54b8f8 100644 --- a/pixels-core/src/main/java/io/pixelsdb/pixels/core/encoding/Encoder.java +++ b/pixels-core/src/main/java/io/pixelsdb/pixels/core/encoding/Encoder.java @@ -79,7 +79,12 @@ public byte[] encode(int[] values, int offset, int length) throws IOException throw new PixelsEncodingException("Encoding int values is not supported"); } - public byte[] encode(short[] values) + public byte[] encode(short[] values) throws IOException + { + throw new PixelsEncodingException("Encoding short values is not supported"); + } + + public byte[] encode(short[] values, int offset, int length) throws IOException { throw new PixelsEncodingException("Encoding short values is not supported"); } diff --git a/pixels-core/src/main/java/io/pixelsdb/pixels/core/encoding/RunLenIntEncoder.java b/pixels-core/src/main/java/io/pixelsdb/pixels/core/encoding/RunLenIntEncoder.java index ec67ea5c34..c46c9ce408 100644 --- a/pixels-core/src/main/java/io/pixelsdb/pixels/core/encoding/RunLenIntEncoder.java +++ b/pixels-core/src/main/java/io/pixelsdb/pixels/core/encoding/RunLenIntEncoder.java @@ -113,6 +113,19 @@ public byte[] encode(int[] values, int offset, int length) throws IOException return result; } + @Override + public byte[] encode(short[] values, int offset, int length) throws IOException + { + for (int i = 0; i < length; i++) + { + this.write(values[i+offset]); + } + flush(); + byte[] result = outputStream.toByteArray(); + outputStream.reset(); + return result; + } + @Override public byte[] encode(long[] values) throws IOException { @@ -125,6 +138,12 @@ public byte[] encode(int[] values) throws IOException return encode(values, 0, values.length); } + @Override + public byte[] encode(short[] values) throws IOException + { + return encode(values, 0, values.length); + } + @Override public void close() throws IOException { diff --git a/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/ColumnReader.java b/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/ColumnReader.java index b7a123fce6..294783f3e2 100644 --- a/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/ColumnReader.java +++ b/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/ColumnReader.java @@ -64,15 +64,23 @@ public static ColumnReader newColumnReader(TypeDescription type, PixelsReaderOpt case BYTE: return new ByteColumnReader(type); case SHORT: - case INT: - if (option.isReadIntColumnAsIntVector()) + if (option.isReadShortColumnAsLongVector()) { - return new IntColumnReader(type); + return new LongColumnReader(type); } else + { + return new ShortColumnReader(type); + } + case INT: + if (option.isReadIntColumnAsLongVector()) { return new LongColumnReader(type); } + else + { + return new IntColumnReader(type); + } case LONG: return new LongColumnReader(type); case DOUBLE: diff --git a/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/IntColumnReader.java b/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/IntColumnReader.java index 98f4dc5297..aa9ccf369b 100644 --- a/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/IntColumnReader.java +++ b/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/IntColumnReader.java @@ -37,14 +37,19 @@ /** * This is the column reader for integer (int32) columns. - * In some query engines (e.g., Trino 405 and Presto 0.279), the integer column should be read into long[] - * (i.e., {@link LongColumnVector}) in memory. In this case, the integer column should be read by - * {@link LongColumnReader} instead of {@link IntColumnReader}. + *

+ * In some query engines (e.g., Trino 405 and Presto 0.279), the integer column + * should be read into {@code long[]} (i.e., {@link LongColumnVector}) in memory. + * In that case, set {@link PixelsReaderOption#readIntColumnAsLongVector(boolean)} + * to {@code true} so that {@link LongColumnReader} is used instead of this reader. + *

+ * However, in some other query engines (e.g., Trino 466), the integer column + * should be read into {@code int[]} (i.e., {@link IntColumnVector}) + * by this reader, which is the default behavior. * - * However, in some other query engines (e.g., Trino 466), the integer column should be read into int[] - * (i.e., {@link IntColumnVector}) by {@link IntColumnReader}. - * @author hank + * @author hank, gengdy * @create 2024-12-02 + * @update 2026-08-07 */ public class IntColumnReader extends ColumnReader { @@ -333,7 +338,7 @@ else if (selected.get(j - vectorIndex)) { for (int j = i; j < i + numToRead; ++j) { - if (!(hasNull && isNull[j])) + if (!(hasNull && isNull[j - vectorIndex])) { int value = inputBuffer.getInt(); if (selected.get(j - vectorIndex)) diff --git a/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/LongColumnReader.java b/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/LongColumnReader.java index 6b26d1b57f..aad1052bd7 100644 --- a/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/LongColumnReader.java +++ b/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/LongColumnReader.java @@ -35,6 +35,10 @@ import java.util.Arrays; /** + * This column reader reads short (int16), integer (int32), and long (int64) columns + * into a {@link LongColumnVector}. It is used by old query engines (e.g. Trino 405, + * Presto 0.279) that expect short and integer columns in {@code long[]}. + * * @author guodong, hank * @create 2017-12-06 * @update 2023-08-21: support nulls padding @@ -46,15 +50,34 @@ public class LongColumnReader extends ColumnReader private InputStream inputStream; /** - * True if the data type of the values is long (int64), otherwise the data type is int32. + * The number of bytes per value when the encoding is NONE (not run-length encoded). + * For SHORT it is 2, for INT it is 4, and for LONG it is 8. When the encoding is + * RUNLENGTH, this field is unused because the decoder always returns a long. */ - private boolean isLong = false; + private int readBytes = Long.BYTES; LongColumnReader(TypeDescription type) { super(type); } + /** + * Reads the next value from the input buffer, widening it to long if the + * underlying column is SHORT (2 bytes) or INT (4 bytes). + */ + private long readNextValue() + { + switch (readBytes) + { + case Short.BYTES: + return inputBuffer.getShort(); + case Integer.BYTES: + return inputBuffer.getInt(); + default: + return inputBuffer.getLong(); + } + } + /** * Closes this column reader and releases any resources associated * with it. If the column reader is already closed then invoking this @@ -120,7 +143,18 @@ public void read(ByteBuffer input, PixelsProto.ColumnEncoding encoding, elementIndex = 0; if (encoding.getKind().equals(PixelsProto.ColumnEncoding.Kind.NONE)) { - isLong = type.getCategory() == TypeDescription.Category.LONG; + switch (type.getCategory()) + { + case SHORT: + readBytes = Short.BYTES; + break; + case INT: + readBytes = Integer.BYTES; + break; + default: + readBytes = Long.BYTES; + break; + } } } @@ -169,43 +203,20 @@ public void read(ByteBuffer input, PixelsProto.ColumnEncoding encoding, } else { - if (isLong) + if (nullsPadding) { - if (nullsPadding) - { - for (int j = i; j < i + numToRead; ++j) - { - columnVector.vector[j] = inputBuffer.getLong(); - } - } - else + for (int j = i; j < i + numToRead; ++j) { - for (int j = i; j < i + numToRead; ++j) - { - if (!(hasNull && columnVector.isNull[j])) - { - columnVector.vector[j] = inputBuffer.getLong(); - } - } + columnVector.vector[j] = readNextValue(); } } else { - if (nullsPadding) - { - for (int j = i; j < i + numToRead; ++j) - { - columnVector.vector[j] = inputBuffer.getInt(); - } - } - else + for (int j = i; j < i + numToRead; ++j) { - for (int j = i; j < i + numToRead; ++j) + if (!(hasNull && columnVector.isNull[j])) { - if (!(hasNull && columnVector.isNull[j])) - { - columnVector.vector[j] = inputBuffer.getInt(); - } + columnVector.vector[j] = readNextValue(); } } } @@ -259,7 +270,18 @@ public void readSelected(ByteBuffer input, PixelsProto.ColumnEncoding encoding, elementIndex = 0; if (encoding.getKind().equals(PixelsProto.ColumnEncoding.Kind.NONE)) { - isLong = type.getCategory() == TypeDescription.Category.LONG; + switch (type.getCategory()) + { + case SHORT: + readBytes = Short.BYTES; + break; + case INT: + readBytes = Integer.BYTES; + break; + default: + readBytes = Long.BYTES; + break; + } } } @@ -347,67 +369,32 @@ else if (selected.get(j - vectorIndex)) } else { - if (isLong) + if (nullsPadding) { - if (nullsPadding) - { - for (int j = i; j < i + numToRead; ++j) - { - long value = inputBuffer.getLong(); - if (selected.get(j - vectorIndex)) - { - columnVector.vector[vectorWriteIndex++] = value; - } - } - } - else + for (int j = i; j < i + numToRead; ++j) { - for (int j = i; j < i + numToRead; ++j) + long value = readNextValue(); + if (selected.get(j - vectorIndex)) { - if (!(hasNull && isNull[j - vectorIndex])) - { - long value = inputBuffer.getLong(); - if (selected.get(j - vectorIndex)) - { - columnVector.vector[vectorWriteIndex++] = value; - } - } - else if (selected.get(j - vectorIndex)) - { - vectorWriteIndex++; - } + columnVector.vector[vectorWriteIndex++] = value; } } } else { - if (nullsPadding) + for (int j = i; j < i + numToRead; ++j) { - for (int j = i; j < i + numToRead; ++j) + if (!(hasNull && isNull[j - vectorIndex])) { - int value = inputBuffer.getInt(); + long value = readNextValue(); if (selected.get(j - vectorIndex)) { columnVector.vector[vectorWriteIndex++] = value; } } - } - else - { - for (int j = i; j < i + numToRead; ++j) + else if (selected.get(j - vectorIndex)) { - if (!(hasNull && isNull[j])) - { - int value = inputBuffer.getInt(); - if (selected.get(j - vectorIndex)) - { - columnVector.vector[vectorWriteIndex++] = value; - } - } - else if (selected.get(j - vectorIndex)) - { - vectorWriteIndex++; - } + vectorWriteIndex++; } } } diff --git a/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/PixelsReaderOption.java b/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/PixelsReaderOption.java index ad176b2f48..0444438dd6 100644 --- a/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/PixelsReaderOption.java +++ b/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/PixelsReaderOption.java @@ -33,7 +33,8 @@ public class PixelsReaderOption private boolean skipCorruptRecords = false; private boolean tolerantSchemaEvolution = true; // this may lead to column missing due to schema evolution private boolean enableEncodedColumnVector = false; // whether read encoded column vectors directly when possible - private boolean readIntColumnAsIntVector = false; // whether read int32 columns as int32 column vectors + private boolean readIntColumnAsLongVector = false; // whether read int32 columns as long column vectors, for backward compatibility of old query engines + private boolean readShortColumnAsLongVector = false; // whether read int16 columns as long column vectors, for backward compatibility of old query engines private boolean exposeHiddenColumn = false; // whether expose the hidden commit timestamp column in the result batch private long transId = -1L; private long transTimestamp = -1L; // -1 means no need to consider the timestamp when reading data @@ -145,14 +146,36 @@ public boolean isEnableEncodedColumnVector() return enableEncodedColumnVector; } - public void readIntColumnAsIntVector(boolean readIntColumnAsIntVector) + /** + * Whether to read int32 (integer) columns as {@link io.pixelsdb.pixels.core.vector.LongColumnVector} + * instead of {@link io.pixelsdb.pixels.core.vector.IntColumnVector}. Old query engines such as + * Trino 405 and Presto 0.279 expect integer columns in {@code long[]}, so they should set this + * to {@code true}. By default, integer columns are read into int vectors. + */ + public void readIntColumnAsLongVector(boolean readIntColumnAsLongVector) { - this.readIntColumnAsIntVector = readIntColumnAsIntVector; + this.readIntColumnAsLongVector = readIntColumnAsLongVector; } - public boolean isReadIntColumnAsIntVector() + public boolean isReadIntColumnAsLongVector() { - return readIntColumnAsIntVector; + return readIntColumnAsLongVector; + } + + /** + * Whether to read int16 (smallint) columns as {@link io.pixelsdb.pixels.core.vector.LongColumnVector} + * instead of {@link io.pixelsdb.pixels.core.vector.ShortColumnVector}. Old query engines such as + * Trino 405 and Presto 0.279 expect short columns in {@code long[]}, so they should set this + * to {@code true}. By default, short columns are read into short vectors. + */ + public void readShortColumnAsLongVector(boolean readShortColumnAsLongVector) + { + this.readShortColumnAsLongVector = readShortColumnAsLongVector; + } + + public boolean isReadShortColumnAsLongVector() + { + return readShortColumnAsLongVector; } public PixelsReaderOption exposeHiddenColumn(boolean exposeHiddenColumn) diff --git a/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/PixelsRecordReaderBufferImpl.java b/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/PixelsRecordReaderBufferImpl.java index 0891ce3081..77f96fec9c 100644 --- a/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/PixelsRecordReaderBufferImpl.java +++ b/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/PixelsRecordReaderBufferImpl.java @@ -58,7 +58,7 @@ public class PixelsRecordReaderBufferImpl implements PixelsRecordReader private final boolean retinaEnabled; private final TypeDescription typeDescription; private final int colNum; - private final int typeMode = TypeDescription.Mode.CREATE_INT_VECTOR_FOR_INT; + private final int vectorLayout; private static ExecutorService prefetchExecutor; // Thread pool for I/O and deserialization private final BlockingQueue prefetchQueue; // Queue for completed batches private final AtomicInteger pendingTasks = new AtomicInteger(0); // Counter for submitted but unfinished tasks @@ -117,6 +117,8 @@ public PixelsRecordReaderBufferImpl(PixelsReaderOption option, this.retinaEnabled = Boolean.parseBoolean(configFactory.getProperty("retina.enable")); this.option = option; + this.vectorLayout = (option.isReadIntColumnAsLongVector() ? TypeDescription.VectorLayout.INT_AS_LONG : 0) | + (option.isReadShortColumnAsLongVector() ? TypeDescription.VectorLayout.SHORT_AS_LONG : 0); this.activeMemtableData = activeMemtableData; this.fileIds = fileIds; this.storage = storage; @@ -328,7 +330,7 @@ public int prepareBatch(int batchSize) throws IOException private VectorizedRowBatch createEmptyRowBatch(int size) { TypeDescription resultSchema = TypeDescription.createSchema(new ArrayList<>()); - VectorizedRowBatch resultRowBatch = resultSchema.createRowBatch(0, this.typeMode); + VectorizedRowBatch resultRowBatch = resultSchema.createRowBatch(0, vectorLayout); resultRowBatch.projectionSize = 0; resultRowBatch.endOfFile = this.endOfFile; resultRowBatch.size = size; diff --git a/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/PixelsRecordReaderImpl.java b/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/PixelsRecordReaderImpl.java index bd1ed826e6..b294e6887c 100644 --- a/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/PixelsRecordReaderImpl.java +++ b/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/PixelsRecordReaderImpl.java @@ -133,7 +133,7 @@ public class PixelsRecordReaderImpl implements PixelsRecordReader private ColumnReader[] readers; // column readers for each target columns private long[][] rgVisibilityBitmaps; // bitmaps of row group visibility private final boolean enableEncodedVector; - private final int typeMode; + private final int vectorLayout; private long diskReadBytes = 0L; private long cacheReadBytes = 0L; @@ -172,8 +172,8 @@ public PixelsRecordReaderImpl(PhysicalReader physicalReader, this.RGStart = option.getRGStart(); this.RGLen = option.getRGLen(); this.enableEncodedVector = option.isEnableEncodedColumnVector(); - this.typeMode = option.isReadIntColumnAsIntVector() ? - TypeDescription.Mode.CREATE_INT_VECTOR_FOR_INT : TypeDescription.Mode.NONE; + this.vectorLayout = (option.isReadIntColumnAsLongVector() ? TypeDescription.VectorLayout.INT_AS_LONG : 0) | + (option.isReadShortColumnAsLongVector() ? TypeDescription.VectorLayout.SHORT_AS_LONG : 0); this.enableMetrics = enableMetrics; this.metricsDir = metricsDir; this.readPerfMetrics = new ReadPerfMetrics(); @@ -1034,7 +1034,7 @@ public int prepareBatch(int batchSize) throws IOException private VectorizedRowBatch createEmptyEOFRowBatch(int size) { TypeDescription resultSchema = TypeDescription.createSchema(new ArrayList<>()); - VectorizedRowBatch resultRowBatch = resultSchema.createRowBatch(0, typeMode); + VectorizedRowBatch resultRowBatch = resultSchema.createRowBatch(0, vectorLayout); resultRowBatch.projectionSize = 0; resultRowBatch.endOfFile = true; resultRowBatch.size = size; @@ -1097,7 +1097,7 @@ public VectorizedRowBatch readBatch(int batchSize, boolean reuse) { if (this.resultRowBatch == null || this.resultRowBatch.projectionSize != resultColumns.length) { - this.resultRowBatch = resultSchema.createRowBatch(batchSize, typeMode, resultColumnsEncoded); + this.resultRowBatch = resultSchema.createRowBatch(batchSize, vectorLayout, resultColumnsEncoded); this.resultRowBatch.projectionSize = resultColumns.length; if (option.isExposeHiddenColumn()) { @@ -1113,7 +1113,7 @@ public VectorizedRowBatch readBatch(int batchSize, boolean reuse) resultRowBatch = this.resultRowBatch; } else { - resultRowBatch = resultSchema.createRowBatch(batchSize, typeMode, resultColumnsEncoded); + resultRowBatch = resultSchema.createRowBatch(batchSize, vectorLayout, resultColumnsEncoded); resultRowBatch.projectionSize = resultColumns.length; if (option.isExposeHiddenColumn()) { diff --git a/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/PixelsRecordReaderStreamImpl.java b/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/PixelsRecordReaderStreamImpl.java index 6ff6e102d9..4626c67a2a 100644 --- a/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/PixelsRecordReaderStreamImpl.java +++ b/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/PixelsRecordReaderStreamImpl.java @@ -90,6 +90,7 @@ public class PixelsRecordReaderStreamImpl implements PixelsRecordReader private ByteBuffer[] chunkBuffers; // buffers of each chunk in current row group, arranged by chunk's column id private ColumnReader[] readers; // column readers for each target columns private final boolean enableEncodedVector = false; + private final int vectorLayout; private long diskReadBytes = 0L; private long readTimeNanos = 0L; private long memoryUsage = 0L; @@ -122,6 +123,8 @@ public PixelsRecordReaderStreamImpl(PhysicalReader physicalReader, this.curRGFooterBuffer = ByteBuffer.allocate(Constants.STREAM_READER_RG_FOOTER_BUFFER_SIZE); this.streamHeader = streamHeader; this.option = option; + this.vectorLayout = (option.isReadIntColumnAsLongVector() ? TypeDescription.VectorLayout.INT_AS_LONG : 0) | + (option.isReadShortColumnAsLongVector() ? TypeDescription.VectorLayout.SHORT_AS_LONG : 0); this.includedColumnTypes = new ArrayList<>(); checkBeforeRead(); } @@ -246,7 +249,7 @@ public int prepareBatch(int batchSize) private VectorizedRowBatch createEmptyEOFRowBatch(int size) { TypeDescription resultSchema = TypeDescription.createSchema(new ArrayList<>()); - VectorizedRowBatch resultRowBatch = resultSchema.createRowBatch(0, TypeDescription.Mode.NONE); + VectorizedRowBatch resultRowBatch = resultSchema.createRowBatch(0, vectorLayout); resultRowBatch.projectionSize = 0; resultRowBatch.endOfFile = true; resultRowBatch.size = size; @@ -268,8 +271,7 @@ public VectorizedRowBatch readBatch(int batchSize, boolean reuse) throws IOExcep { if (this.resultRowBatch == null || this.resultRowBatch.projectionSize != includedColumnNum) { - this.resultRowBatch = resultSchema.createRowBatch(batchSize, - TypeDescription.Mode.NONE, resultColumnsEncoded); + this.resultRowBatch = resultSchema.createRowBatch(batchSize, vectorLayout, resultColumnsEncoded); this.resultRowBatch.projectionSize = includedColumnNum; } this.resultRowBatch.reset(); @@ -278,7 +280,7 @@ public VectorizedRowBatch readBatch(int batchSize, boolean reuse) throws IOExcep } else { - resultRowBatch = resultSchema.createRowBatch(batchSize, TypeDescription.Mode.NONE, resultColumnsEncoded); + resultRowBatch = resultSchema.createRowBatch(batchSize, vectorLayout, resultColumnsEncoded); resultRowBatch.projectionSize = includedColumnNum; } diff --git a/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/ShortColumnReader.java b/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/ShortColumnReader.java new file mode 100644 index 0000000000..216b81b313 --- /dev/null +++ b/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/ShortColumnReader.java @@ -0,0 +1,300 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Pixels. If not, see + * . + */ +package io.pixelsdb.pixels.core.reader; + +import io.pixelsdb.pixels.core.PixelsProto; +import io.pixelsdb.pixels.core.TypeDescription; +import io.pixelsdb.pixels.core.encoding.RunLenIntDecoder; +import io.pixelsdb.pixels.core.utils.BitUtils; +import io.pixelsdb.pixels.core.utils.Bitmap; +import io.pixelsdb.pixels.core.utils.ByteBufferInputStream; +import io.pixelsdb.pixels.core.vector.ColumnVector; +import io.pixelsdb.pixels.core.vector.ShortColumnVector; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Arrays; + +/** + * This is the column reader for short (int16) columns. + *

+ * In some query engines (e.g., Trino 405 and Presto 0.279), the short column + * should be read into {@code long[]} (i.e., {@link LongColumnVector}) in memory. + * In that case, set {@link PixelsReaderOption#readShortColumnAsLongVector(boolean)} + * to {@code true} so that {@link LongColumnReader} is used instead of this reader. + *

+ * However, in some other query engines (e.g., Trino 466), the short column + * should be read into {@code short[]} (i.e., {@link ShortColumnVector}) + * by this reader, which is the default behavior. + * + * @author gengdy + * @create 2026-08-07 + */ +public class ShortColumnReader extends ColumnReader +{ + private RunLenIntDecoder decoder; + private ByteBuffer inputBuffer; + private InputStream inputStream; + + ShortColumnReader(TypeDescription type) + { + super(type); + } + + @Override + public void close() throws IOException + { + if (this.decoder != null) + { + this.decoder.close(); + this.decoder = null; + } + this.inputBuffer = null; + } + + @Override + public void read(ByteBuffer input, PixelsProto.ColumnEncoding encoding, + int offset, int size, int pixelStride, final int vectorIndex, + ColumnVector vector, PixelsProto.ColumnChunkIndex chunkIndex) throws IOException + { + ShortColumnVector columnVector = (ShortColumnVector) vector; + boolean decoding = encoding.getKind().equals(PixelsProto.ColumnEncoding.Kind.RUNLENGTH); + boolean nullsPadding = chunkIndex.hasNullsPadding() && chunkIndex.getNullsPadding(); + boolean littleEndian = chunkIndex.hasLittleEndian() && chunkIndex.getLittleEndian(); + if (offset == 0) + { + if (inputStream != null) + { + inputStream.close(); + } + this.inputBuffer = input; + this.inputBuffer.order(littleEndian ? ByteOrder.LITTLE_ENDIAN : ByteOrder.BIG_ENDIAN); + inputStream = new ByteBufferInputStream(inputBuffer, inputBuffer.position(), inputBuffer.limit()); + decoder = new RunLenIntDecoder(inputStream, true); + isNullOffset = inputBuffer.position() + chunkIndex.getIsNullOffset(); + isNullSkipBits = 0; + hasNull = true; + elementIndex = 0; + } + + int numLeft = size, numToRead, bytesToDeCompact; + boolean endOfPixel; + for (int i = vectorIndex; numLeft > 0; ) + { + if (elementIndex / pixelStride < (elementIndex + numLeft) / pixelStride) + { + numToRead = pixelStride - elementIndex % pixelStride; + endOfPixel = true; + } + else + { + numToRead = numLeft; + endOfPixel = false; + } + bytesToDeCompact = (numToRead + isNullSkipBits + (endOfPixel ? 7 : 0)) / 8; + int pixelId = elementIndex / pixelStride; + hasNull = chunkIndex.getPixelStatistics(pixelId).getStatistic().getHasNull(); + if (hasNull) + { + BitUtils.bitWiseDeCompact(columnVector.isNull, i, numToRead, + inputBuffer, isNullOffset, isNullSkipBits, littleEndian); + isNullOffset += bytesToDeCompact; + isNullSkipBits = endOfPixel ? 0 : (numToRead + isNullSkipBits) % 8; + columnVector.noNulls = false; + } + else + { + Arrays.fill(columnVector.isNull, i, i + numToRead, false); + } + + if (decoding) + { + for (int j = i; j < i + numToRead; ++j) + { + if (!(hasNull && columnVector.isNull[j])) + { + columnVector.vector[j] = (short) decoder.next(); + } + } + } + else if (nullsPadding) + { + for (int j = i; j < i + numToRead; ++j) + { + columnVector.vector[j] = inputBuffer.getShort(); + } + } + else + { + for (int j = i; j < i + numToRead; ++j) + { + if (!(hasNull && columnVector.isNull[j])) + { + columnVector.vector[j] = inputBuffer.getShort(); + } + } + } + numLeft -= numToRead; + elementIndex += numToRead; + i += numToRead; + } + } + + @Override + public void readSelected(ByteBuffer input, PixelsProto.ColumnEncoding encoding, + int offset, int size, int pixelStride, final int vectorIndex, + ColumnVector vector, PixelsProto.ColumnChunkIndex chunkIndex, + Bitmap selected) throws IOException + { + ShortColumnVector columnVector = (ShortColumnVector) vector; + boolean decoding = encoding.getKind().equals(PixelsProto.ColumnEncoding.Kind.RUNLENGTH); + boolean nullsPadding = chunkIndex.hasNullsPadding() && chunkIndex.getNullsPadding(); + boolean littleEndian = chunkIndex.hasLittleEndian() && chunkIndex.getLittleEndian(); + if (offset == 0) + { + if (inputStream != null) + { + inputStream.close(); + } + this.inputBuffer = input; + this.inputBuffer.order(littleEndian ? ByteOrder.LITTLE_ENDIAN : ByteOrder.BIG_ENDIAN); + inputStream = new ByteBufferInputStream(inputBuffer, inputBuffer.position(), inputBuffer.limit()); + decoder = new RunLenIntDecoder(inputStream, true); + isNullOffset = inputBuffer.position() + chunkIndex.getIsNullOffset(); + isNullSkipBits = 0; + hasNull = true; + elementIndex = 0; + } + + int numLeft = size, numToRead, bytesToDeCompact, vectorWriteIndex = vectorIndex; + boolean[] isNull = null; + boolean endOfPixel; + if (decoding || !nullsPadding) + { + isNull = new boolean[size]; + } + for (int i = vectorIndex; numLeft > 0; ) + { + if (elementIndex / pixelStride < (elementIndex + numLeft) / pixelStride) + { + numToRead = pixelStride - elementIndex % pixelStride; + endOfPixel = true; + } + else + { + numToRead = numLeft; + endOfPixel = false; + } + bytesToDeCompact = (numToRead + isNullSkipBits + (endOfPixel ? 7 : 0)) / 8; + int pixelId = elementIndex / pixelStride; + hasNull = chunkIndex.getPixelStatistics(pixelId).getStatistic().getHasNull(); + if (hasNull) + { + if (!decoding && nullsPadding) + { + BitUtils.bitWiseDeCompact(columnVector.isNull, vectorWriteIndex, numToRead, inputBuffer, + isNullOffset, isNullSkipBits, littleEndian, selected, i - vectorIndex); + } + else + { + BitUtils.bitWiseDeCompact(isNull, i - vectorIndex, numToRead, inputBuffer, + isNullOffset, isNullSkipBits, littleEndian); + int k = vectorWriteIndex; + for (int j = i; j < i + numToRead; ++j) + { + if (selected.get(j - vectorIndex)) + { + columnVector.isNull[k++] = isNull[j - vectorIndex]; + } + } + } + isNullOffset += bytesToDeCompact; + isNullSkipBits = endOfPixel ? 0 : (numToRead + isNullSkipBits) % 8; + columnVector.noNulls = false; + } + else + { + if (decoding || !nullsPadding) + { + Arrays.fill(isNull, i - vectorIndex, i - vectorIndex + numToRead, false); + } + } + + int originalVectorWriteIndex = vectorWriteIndex; + if (decoding) + { + for (int j = i; j < i + numToRead; ++j) + { + if (!(hasNull && isNull[j - vectorIndex])) + { + short value = (short) decoder.next(); + if (selected.get(j - vectorIndex)) + { + columnVector.vector[vectorWriteIndex++] = value; + } + } + else if (selected.get(j - vectorIndex)) + { + vectorWriteIndex++; + } + } + } + else if (nullsPadding) + { + for (int j = i; j < i + numToRead; ++j) + { + short value = inputBuffer.getShort(); + if (selected.get(j - vectorIndex)) + { + columnVector.vector[vectorWriteIndex++] = value; + } + } + } + else + { + for (int j = i; j < i + numToRead; ++j) + { + if (!(hasNull && isNull[j - vectorIndex])) + { + short value = inputBuffer.getShort(); + if (selected.get(j - vectorIndex)) + { + columnVector.vector[vectorWriteIndex++] = value; + } + } + else if (selected.get(j - vectorIndex)) + { + vectorWriteIndex++; + } + } + } + + if (!hasNull) + { + Arrays.fill(columnVector.isNull, originalVectorWriteIndex, vectorWriteIndex, false); + } + numLeft -= numToRead; + elementIndex += numToRead; + i += numToRead; + } + } +} diff --git a/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/VectorColumnReader.java b/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/VectorColumnReader.java index e0a3415799..61391cd0a6 100644 --- a/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/VectorColumnReader.java +++ b/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/VectorColumnReader.java @@ -60,7 +60,13 @@ public void read(ByteBuffer input, PixelsProto.ColumnEncoding encoding, int offs int vectorIndex, ColumnVector vector, PixelsProto.ColumnChunkIndex chunkIndex) throws IOException { VectorColumnVector vectorColumnVector = (VectorColumnVector) vector; - ((VectorColumnVector) vector).vector = new double[vector.getLength()][((VectorColumnVector) vector).dimension]; + for (int i = vectorIndex; i < vectorIndex + size; ++i) + { + if (vectorColumnVector.vector[i] == null) + { + vectorColumnVector.vector[i] = new double[vectorColumnVector.dimension]; + } + } boolean nullsPadding = chunkIndex.hasNullsPadding() && chunkIndex.getNullsPadding(); boolean littleEndian = chunkIndex.hasLittleEndian() && chunkIndex.getLittleEndian(); if (offset == 0) @@ -167,7 +173,10 @@ public void readSelected(ByteBuffer input, PixelsProto.ColumnEncoding encoding, // keep origin content for (int i = vectorIndex; i < vector.getLength(); ++i) { - ((VectorColumnVector) vector).vector[i] = new double[((VectorColumnVector) vector).dimension]; + if (vectorColumnVector.vector[i] == null) + { + vectorColumnVector.vector[i] = new double[vectorColumnVector.dimension]; + } } boolean nullsPadding = chunkIndex.hasNullsPadding() && chunkIndex.getNullsPadding(); boolean littleEndian = chunkIndex.hasLittleEndian() && chunkIndex.getLittleEndian(); diff --git a/pixels-core/src/main/java/io/pixelsdb/pixels/core/stats/IntegerStatsRecorder.java b/pixels-core/src/main/java/io/pixelsdb/pixels/core/stats/IntegerStatsRecorder.java index c064328a4a..73dfaf8800 100644 --- a/pixels-core/src/main/java/io/pixelsdb/pixels/core/stats/IntegerStatsRecorder.java +++ b/pixels-core/src/main/java/io/pixelsdb/pixels/core/stats/IntegerStatsRecorder.java @@ -219,11 +219,11 @@ public double getSelectivity(Object lowerBound, boolean lowerInclusive, Object u long upper = maximum; if (lowerBound != null) { - lower = (long) lowerBound; + lower = ((Number) lowerBound).longValue(); } if (upperBound != null) { - upper = (long) upperBound; + upper = ((Number) upperBound).longValue(); } checkArgument(lower <= upper, "lower bound must be larger than the upper bound"); if (lower < minimum) diff --git a/pixels-core/src/main/java/io/pixelsdb/pixels/core/vector/IntColumnVector.java b/pixels-core/src/main/java/io/pixelsdb/pixels/core/vector/IntColumnVector.java index 7d1542ca5d..a3af65fb15 100644 --- a/pixels-core/src/main/java/io/pixelsdb/pixels/core/vector/IntColumnVector.java +++ b/pixels-core/src/main/java/io/pixelsdb/pixels/core/vector/IntColumnVector.java @@ -32,7 +32,7 @@ /** * This class represents a nullable int column vector. - * This class uses a 32-bit long value to hold the values. + * This class uses a 32-bit integer value to hold the values. * In high Java versions such as Java 23, 32-bit integer has comparable operation performance as 64-bit integer. * Therefore, using 32-bit column vector for int32 columns saves memory without performance degradation. *

@@ -64,7 +64,7 @@ public IntColumnVector(int len) { super(len); vector = new int[len]; - memoryUsage += Integer.BYTES * len; + memoryUsage += (long) Integer.BYTES * len; } @Override @@ -213,7 +213,7 @@ public void addElement(int inputIndex, ColumnVector inputVector) @Override public void addSelected(int[] selected, int offset, int length, ColumnVector src) { - // isRepeating should be false and src should be an instance of LongColumnVector. + // isRepeating should be false and src should be an instance of IntColumnVector. // However, we do not check these for performance considerations. IntColumnVector source = (IntColumnVector) src; diff --git a/pixels-core/src/main/java/io/pixelsdb/pixels/core/vector/ShortColumnVector.java b/pixels-core/src/main/java/io/pixelsdb/pixels/core/vector/ShortColumnVector.java new file mode 100644 index 0000000000..11595a103f --- /dev/null +++ b/pixels-core/src/main/java/io/pixelsdb/pixels/core/vector/ShortColumnVector.java @@ -0,0 +1,341 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the Affero GNU General Public + * License along with Pixels. If not, see + * . + */ +package io.pixelsdb.pixels.core.vector; + +import com.google.flatbuffers.FlatBufferBuilder; +import io.pixelsdb.pixels.core.flat.ColumnVectorFlat; +import io.pixelsdb.pixels.core.flat.ShortColumnVectorFlat; +import io.pixelsdb.pixels.core.utils.Bitmap; + +import java.nio.ByteBuffer; +import java.util.Arrays; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +/** + * This class represents a nullable short (int16) column vector. + * This class uses 16-bit integer values. + * + * @author gengdy + * @create 2026-08-07 + */ +public class ShortColumnVector extends ColumnVector +{ + public short[] vector; + + public ShortColumnVector() + { + this(VectorizedRowBatch.DEFAULT_SIZE); + } + + public ShortColumnVector(int len) + { + super(len); + this.vector = new short[len]; + this.memoryUsage += (long) Short.BYTES * len; + } + + @Override + public void add(int value) + { + add((long) value); + } + + @Override + public void add(long value) + { + if (value < Short.MIN_VALUE || value > Short.MAX_VALUE) + { + throw new IllegalArgumentException("SHORT value out of range: " + value); + } + add((short) value); + } + + @Override + public void add(String value) + { + switch (value.toLowerCase()) + { + case "true": + add(1); + break; + case "false": + add(0); + break; + default: + add(Short.parseShort(value)); + break; + } + } + + @Override + public void add(boolean value) + { + add(value ? 1 : 0); + } + + public void add(short value) + { + if (writeIndex >= getLength()) + { + ensureSize(writeIndex * 2, true); + } + int index = writeIndex++; + this.vector[index] = value; + this.isNull[index] = false; + } + + @Override + public void add(byte[] value) + { + if (checkBytesNull(value)) + { + return; + } + if (value.length != Short.BYTES) + { + throw new IllegalArgumentException("Only byte[2] supported for serialization to short"); + } + short v = ByteBuffer.wrap(value).getShort(); + add(v); + } + + @Override + public int[] accumulateHashCode(int[] hashCode) + { + requireNonNull(hashCode, "hashCode is null"); + checkArgument(hashCode.length > 0 && hashCode.length <= this.length, + "the length of hashCode is not in the range [1, length]"); + for (int i = 0; i < hashCode.length; ++i) + { + if (!this.isNull[i]) + { + int value = this.vector[i]; + hashCode[i] = 31 * hashCode[i] + (value ^ (value >>> 16)); + } + } + return hashCode; + } + + @Override + public boolean elementEquals(int index, int otherIndex, ColumnVector other) + { + ShortColumnVector otherVector = (ShortColumnVector) other; + return !this.isNull[index] && !otherVector.isNull[otherIndex] && + this.vector[index] == otherVector.vector[otherIndex]; + } + + @Override + public int compareElement(int index, int otherIndex, ColumnVector other) + { + ShortColumnVector otherVector = (ShortColumnVector) other; + if (!this.isNull[index] && !otherVector.isNull[otherIndex]) + { + return Short.compare(this.vector[index], otherVector.vector[otherIndex]); + } + return this.isNull[index] ? -1 : 1; + } + + public void fill(short value) + { + this.noNulls = true; + this.isRepeating = true; + this.vector[0] = value; + } + + @Override + public void flatten(boolean selectedInUse, int[] sel, int size) + { + flattenPush(); + if (isRepeating) + { + isRepeating = false; + short repeatValue = vector[0]; + if (selectedInUse) + { + for (int j = 0; j < size; ++j) + { + vector[sel[j]] = repeatValue; + } + } + else + { + Arrays.fill(vector, 0, size, repeatValue); + } + writeIndex = size; + flattenRepeatingNulls(selectedInUse, sel, size); + } + flattenNoNulls(selectedInUse, sel, size); + } + + @Override + public void addElement(int inputIndex, ColumnVector inputVector) + { + int index = writeIndex++; + if (inputVector.noNulls || !inputVector.isNull[inputIndex]) + { + this.isNull[index] = false; + this.vector[index] = ((ShortColumnVector) inputVector).vector[inputIndex]; + } + else + { + this.isNull[index] = true; + this.noNulls = false; + } + } + + @Override + public void addSelected(int[] selected, int offset, int length, ColumnVector src) + { + ShortColumnVector source = (ShortColumnVector) src; + for (int i = offset; i < offset + length; ++i) + { + int sourceIndex = selected[i]; + int targetIndex = writeIndex++; + if (source.isNull[sourceIndex]) + { + this.isNull[targetIndex] = true; + this.noNulls = false; + } + else + { + this.vector[targetIndex] = source.vector[sourceIndex]; + this.isNull[targetIndex] = false; + } + } + } + + @Override + public void duplicate(ColumnVector inputVector) + { + if (inputVector instanceof ShortColumnVector) + { + ShortColumnVector source = (ShortColumnVector) inputVector; + this.vector = source.vector; + this.isNull = source.isNull; + this.writeIndex = source.writeIndex; + this.noNulls = source.noNulls; + this.isRepeating = source.isRepeating; + } + } + + @Override + protected void applyFilter(Bitmap filter, int before) + { + checkArgument(!isRepeating, + "column vector is repeating, flatten before applying filter"); + checkArgument(before > 0 && before <= length, + "before index is not in the range [1, length]"); + boolean filteredNoNulls = true; + int targetIndex = 0; + for (int sourceIndex = filter.nextSetBit(0); + sourceIndex >= 0 && sourceIndex < before; + sourceIndex = filter.nextSetBit(sourceIndex + 1), ++targetIndex) + { + if (sourceIndex > targetIndex) + { + this.vector[targetIndex] = this.vector[sourceIndex]; + this.isNull[targetIndex] = this.isNull[sourceIndex]; + } + if (this.isNull[targetIndex]) + { + filteredNoNulls = false; + } + } + this.noNulls = filteredNoNulls; + } + + @Override + public void stringifyValue(StringBuilder buffer, int row) + { + if (isRepeating) + { + row = 0; + } + if (noNulls || !isNull[row]) + { + buffer.append(vector[row]); + } + else + { + buffer.append("null"); + } + } + + @Override + public void ensureSize(int size, boolean preserveData) + { + super.ensureSize(size, preserveData); + if (size > vector.length) + { + short[] oldArray = vector; + vector = new short[size]; + memoryUsage += (long) Short.BYTES * size; + length = size; + if (preserveData) + { + if (isRepeating) + { + vector[0] = oldArray[0]; + } + else + { + System.arraycopy(oldArray, 0, vector, 0, oldArray.length); + } + } + } + } + + @Override + public void close() + { + super.close(); + this.vector = null; + } + + @Override + public byte getFlatBufferType() + { + return ColumnVectorFlat.ShortColumnVectorFlat; + } + + @Override + public int serialize(FlatBufferBuilder builder) + { + int baseOffset = super.serialize(builder); + int vectorOffset = ShortColumnVectorFlat.createVectorVector(builder, vector); + ShortColumnVectorFlat.startShortColumnVectorFlat(builder); + ShortColumnVectorFlat.addBase(builder, baseOffset); + ShortColumnVectorFlat.addVector(builder, vectorOffset); + return ShortColumnVectorFlat.endShortColumnVectorFlat(builder); + } + + public static ShortColumnVector deserialize(ShortColumnVectorFlat flat) + { + ShortColumnVector result = new ShortColumnVector(flat.base().length()); + for (int i = 0; i < flat.vectorLength(); ++i) + { + result.vector[i] = flat.vector(i); + } + result.deserializeBase(flat.base()); + return result; + } +} diff --git a/pixels-core/src/main/java/io/pixelsdb/pixels/core/vector/VectorizedRowBatch.java b/pixels-core/src/main/java/io/pixelsdb/pixels/core/vector/VectorizedRowBatch.java index ebc6960281..6ded9a50b7 100644 --- a/pixels-core/src/main/java/io/pixelsdb/pixels/core/vector/VectorizedRowBatch.java +++ b/pixels-core/src/main/java/io/pixelsdb/pixels/core/vector/VectorizedRowBatch.java @@ -20,7 +20,6 @@ package io.pixelsdb.pixels.core.vector; import com.google.flatbuffers.FlatBufferBuilder; -import com.google.flatbuffers.Table; import io.pixelsdb.pixels.core.flat.*; import io.pixelsdb.pixels.core.utils.Bitmap; @@ -426,7 +425,6 @@ public static VectorizedRowBatch deserialize(ByteBuffer buffer) for (int i = 0; i < batchFlat.numCols(); ++i) { - Table colTable; switch (batchFlat.colsType(i)) { case ColumnVectorFlat.BinaryColumnVectorFlat: @@ -453,6 +451,9 @@ public static VectorizedRowBatch deserialize(ByteBuffer buffer) case ColumnVectorFlat.IntColumnVectorFlat: batch.cols[i] = IntColumnVector.deserialize((IntColumnVectorFlat) batchFlat.cols(new IntColumnVectorFlat(), i)); break; + case ColumnVectorFlat.ShortColumnVectorFlat: + batch.cols[i] = ShortColumnVector.deserialize((ShortColumnVectorFlat) batchFlat.cols(new ShortColumnVectorFlat(), i)); + break; case ColumnVectorFlat.LongColumnVectorFlat: batch.cols[i] = LongColumnVector.deserialize((LongColumnVectorFlat) batchFlat.cols(new LongColumnVectorFlat(), i)); break; diff --git a/pixels-core/src/main/java/io/pixelsdb/pixels/core/writer/ColumnWriter.java b/pixels-core/src/main/java/io/pixelsdb/pixels/core/writer/ColumnWriter.java index c1a7064def..289defc420 100644 --- a/pixels-core/src/main/java/io/pixelsdb/pixels/core/writer/ColumnWriter.java +++ b/pixels-core/src/main/java/io/pixelsdb/pixels/core/writer/ColumnWriter.java @@ -51,9 +51,11 @@ static ColumnWriter newColumnWriter(TypeDescription type, PixelsWriterOption wri case BYTE: return new ByteColumnWriter(type, writerOption); case SHORT: + return new ShortColumnWriter(type, writerOption); case INT: + return new IntColumnWriter(type, writerOption); case LONG: - return new IntegerColumnWriter(type, writerOption); + return new LongColumnWriter(type, writerOption); case FLOAT: return new FloatColumnWriter(type, writerOption); case DOUBLE: diff --git a/pixels-core/src/main/java/io/pixelsdb/pixels/core/writer/IntColumnWriter.java b/pixels-core/src/main/java/io/pixelsdb/pixels/core/writer/IntColumnWriter.java new file mode 100644 index 0000000000..9f59ce91b6 --- /dev/null +++ b/pixels-core/src/main/java/io/pixelsdb/pixels/core/writer/IntColumnWriter.java @@ -0,0 +1,141 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the Affero GNU General Public License + * along with Pixels. If not, see + * . + */ +package io.pixelsdb.pixels.core.writer; + +import io.pixelsdb.pixels.core.PixelsProto; +import io.pixelsdb.pixels.core.TypeDescription; +import io.pixelsdb.pixels.core.encoding.EncodingLevel; +import io.pixelsdb.pixels.core.encoding.RunLenIntEncoder; +import io.pixelsdb.pixels.core.vector.ColumnVector; +import io.pixelsdb.pixels.core.vector.IntColumnVector; + +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * This is the column writer for integer (int32) columns. + * + * @author gengdy + * @create 2026-08-07 + */ +public class IntColumnWriter extends BaseColumnWriter +{ + private final int[] curPixelVector = new int[pixelStride]; + private final boolean runlengthEncoding; + + public IntColumnWriter(TypeDescription type, PixelsWriterOption writerOption) + { + super(type, writerOption); + runlengthEncoding = encodingLevel.ge(EncodingLevel.EL2); + if (runlengthEncoding) + { + encoder = new RunLenIntEncoder(); + } + } + + @Override + public int write(ColumnVector vector, int size) throws IOException + { + IntColumnVector columnVector = (IntColumnVector) vector; + int curPartLength; + int curPartOffset = 0; + int nextPartLength = size; + while (curPixelIsNullIndex + nextPartLength >= pixelStride) + { + curPartLength = pixelStride - curPixelIsNullIndex; + writeCurPart(columnVector, curPartLength, curPartOffset); + newPixel(); + curPartOffset += curPartLength; + nextPartLength = size - curPartOffset; + } + writeCurPart(columnVector, nextPartLength, curPartOffset); + return outputStream.size(); + } + + private void writeCurPart(IntColumnVector vector, int length, int offset) + { + for (int i = 0; i < length; i++) + { + curPixelEleIndex++; + if (vector.isNull[i + offset]) + { + hasNull = true; + pixelStatRecorder.increment(); + if (nullsPadding) + { + curPixelVector[curPixelVectorIndex++] = 0; + } + } + else + { + curPixelVector[curPixelVectorIndex++] = vector.vector[i + offset]; + } + } + System.arraycopy(vector.isNull, offset, isNull, curPixelIsNullIndex, length); + curPixelIsNullIndex += length; + } + + @Override + void newPixel() throws IOException + { + if (runlengthEncoding) + { + for (int i = 0; i < curPixelVectorIndex; i++) + { + pixelStatRecorder.updateInteger(curPixelVector[i], 1); + } + outputStream.write(encoder.encode(curPixelVector, 0, curPixelVectorIndex)); + } + else + { + ByteBuffer buffer = ByteBuffer.allocate(curPixelVectorIndex * Integer.BYTES).order(byteOrder); + for (int i = 0; i < curPixelVectorIndex; i++) + { + buffer.putInt(curPixelVector[i]); + pixelStatRecorder.updateInteger(curPixelVector[i], 1); + } + outputStream.write(buffer.array()); + } + super.newPixel(); + } + + @Override + public PixelsProto.ColumnEncoding.Builder getColumnChunkEncoding() + { + return PixelsProto.ColumnEncoding.newBuilder().setKind(runlengthEncoding ? + PixelsProto.ColumnEncoding.Kind.RUNLENGTH : PixelsProto.ColumnEncoding.Kind.NONE); + } + + @Override + public void close() throws IOException + { + if (runlengthEncoding) + { + encoder.close(); + } + super.close(); + } + + @Override + public boolean decideNullsPadding(PixelsWriterOption writerOption) + { + return !writerOption.getEncodingLevel().ge(EncodingLevel.EL2) && writerOption.isNullsPadding(); + } +} diff --git a/pixels-core/src/main/java/io/pixelsdb/pixels/core/writer/IntegerColumnWriter.java b/pixels-core/src/main/java/io/pixelsdb/pixels/core/writer/IntegerColumnWriter.java deleted file mode 100644 index b3f701df5b..0000000000 --- a/pixels-core/src/main/java/io/pixelsdb/pixels/core/writer/IntegerColumnWriter.java +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Copyright 2017-2019 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ -package io.pixelsdb.pixels.core.writer; - -import io.pixelsdb.pixels.core.PixelsProto; -import io.pixelsdb.pixels.core.TypeDescription; -import io.pixelsdb.pixels.core.encoding.EncodingLevel; -import io.pixelsdb.pixels.core.encoding.RunLenIntEncoder; -import io.pixelsdb.pixels.core.vector.ColumnVector; -import io.pixelsdb.pixels.core.vector.IntColumnVector; -import io.pixelsdb.pixels.core.vector.LongColumnVector; - -import java.io.IOException; -import java.nio.ByteBuffer; - -/** - * The column writer for integers. - * - * @author guodong, hank - * @update 2023-08-16 Chamonix: support nulls padding - */ -public class IntegerColumnWriter extends BaseColumnWriter -{ - private final long[] curPixelVector = new long[pixelStride]; // current pixel value vector haven't written out yet - private final boolean isLong; // current column type is long or int, used for the first pixel - private final boolean runlengthEncoding; - private int typeDescriptionMode; - public IntegerColumnWriter(TypeDescription type, PixelsWriterOption writerOption) - { - super(type, writerOption); - isLong = type.getCategory() == TypeDescription.Category.LONG; - runlengthEncoding = encodingLevel.ge(EncodingLevel.EL2); - if (runlengthEncoding) - { - encoder = new RunLenIntEncoder(); - } - } - - @Override - public int write(ColumnVector vector, int size) throws IOException - { - if(vector instanceof LongColumnVector) - { - LongColumnVector longColumnVector = (LongColumnVector) vector; - typeDescriptionMode = TypeDescription.Mode.NONE; - writeVector(vector, size, (i, offset) -> longColumnVector.vector[i + offset]); - } else if (vector instanceof IntColumnVector) - { - IntColumnVector intColumnVector = (IntColumnVector) vector; - writeVector(vector, size, (i, offset) -> (long) intColumnVector.vector[i + offset]); - } else - { - throw new IllegalArgumentException("Unsupported ColumnVector type: " + vector.getClass().getName()); - } - - return outputStream.size(); - } - - @FunctionalInterface - private interface ValueAccessor { - long get(int i, int offset); - } - - private void writeVector(ColumnVector columnVector, int size, ValueAccessor accessor) throws IOException - { - int curPartLength; // size of the partition which belongs to current pixel - int curPartOffset = 0; // starting offset of the partition which belongs to current pixel - int nextPartLength = size; // size of the partition which belongs to next pixel - - // do the calculation to partition the vector into current pixel and next one - // doing this pre-calculation to eliminate branch prediction inside the for loop - while ((curPixelIsNullIndex + nextPartLength) >= pixelStride) - { - curPartLength = pixelStride - curPixelIsNullIndex; - writeCurPartLong(columnVector, accessor, curPartLength, curPartOffset); - newPixel(); - curPartOffset += curPartLength; - nextPartLength = size - curPartOffset; - } - - curPartLength = nextPartLength; - writeCurPartLong(columnVector, accessor, curPartLength, curPartOffset); - } - - private void writeCurPartLong(ColumnVector columnVector, ValueAccessor accessor, int curPartLength, int curPartOffset) - { - for (int i = 0; i < curPartLength; i++) - { - curPixelEleIndex++; - if (columnVector.isNull[i + curPartOffset]) - { - hasNull = true; - pixelStatRecorder.increment(); - if (nullsPadding) - { - // padding 0 for nulls - curPixelVector[curPixelVectorIndex++] = 0L; - } - } - else - { - curPixelVector[curPixelVectorIndex++] = accessor.get(i, curPartOffset); - } - } - System.arraycopy(columnVector.isNull, curPartOffset, isNull, curPixelIsNullIndex, curPartLength); - curPixelIsNullIndex += curPartLength; - } - - @Override - void newPixel() throws IOException - { - // write out current pixel vector - if (runlengthEncoding) - { - for (int i = 0; i < curPixelVectorIndex; i++) - { - pixelStatRecorder.updateInteger(curPixelVector[i], 1); - } - outputStream.write(encoder.encode(curPixelVector, 0, curPixelVectorIndex)); - } - else - { - ByteBuffer curVecPartitionBuffer; - if (isLong) - { - curVecPartitionBuffer = ByteBuffer.allocate(curPixelVectorIndex * Long.BYTES); - curVecPartitionBuffer.order(byteOrder); - for (int i = 0; i < curPixelVectorIndex; i++) - { - curVecPartitionBuffer.putLong(curPixelVector[i]); - pixelStatRecorder.updateInteger(curPixelVector[i], 1); - } - } - else - { - curVecPartitionBuffer = ByteBuffer.allocate(curPixelVectorIndex * Integer.BYTES); - curVecPartitionBuffer.order(byteOrder); - for (int i = 0; i < curPixelVectorIndex; i++) - { - curVecPartitionBuffer.putInt((int) curPixelVector[i]); - pixelStatRecorder.updateInteger(curPixelVector[i], 1); - } - } - outputStream.write(curVecPartitionBuffer.array()); - } - - super.newPixel(); - } - - @Override - public PixelsProto.ColumnEncoding.Builder getColumnChunkEncoding() - { - if (runlengthEncoding) - { - return PixelsProto.ColumnEncoding.newBuilder() - .setKind(PixelsProto.ColumnEncoding.Kind.RUNLENGTH); - } - return PixelsProto.ColumnEncoding.newBuilder() - .setKind(PixelsProto.ColumnEncoding.Kind.NONE); - } - - @Override - public void close() throws IOException - { - if (runlengthEncoding) - { - encoder.close(); - } - super.close(); - } - - @Override - public boolean decideNullsPadding(PixelsWriterOption writerOption) - { - if (writerOption.getEncodingLevel().ge(EncodingLevel.EL2)) - { - return false; - } - return writerOption.isNullsPadding(); - } -} diff --git a/pixels-core/src/main/java/io/pixelsdb/pixels/core/writer/LongColumnWriter.java b/pixels-core/src/main/java/io/pixelsdb/pixels/core/writer/LongColumnWriter.java new file mode 100644 index 0000000000..43c002258a --- /dev/null +++ b/pixels-core/src/main/java/io/pixelsdb/pixels/core/writer/LongColumnWriter.java @@ -0,0 +1,141 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the Affero GNU General Public License + * along with Pixels. If not, see + * . + */ +package io.pixelsdb.pixels.core.writer; + +import io.pixelsdb.pixels.core.PixelsProto; +import io.pixelsdb.pixels.core.TypeDescription; +import io.pixelsdb.pixels.core.encoding.EncodingLevel; +import io.pixelsdb.pixels.core.encoding.RunLenIntEncoder; +import io.pixelsdb.pixels.core.vector.ColumnVector; +import io.pixelsdb.pixels.core.vector.LongColumnVector; + +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * This is the column writer for long (int64) columns. + * + * @author gengdy + * @created 2026-08-07 + */ +public class LongColumnWriter extends BaseColumnWriter +{ + private final long[] curPixelVector = new long[pixelStride]; + private final boolean runlengthEncoding; + + public LongColumnWriter(TypeDescription type, PixelsWriterOption writerOption) + { + super(type, writerOption); + runlengthEncoding = encodingLevel.ge(EncodingLevel.EL2); + if (runlengthEncoding) + { + encoder = new RunLenIntEncoder(); + } + } + + @Override + public int write(ColumnVector vector, int size) throws IOException + { + LongColumnVector columnVector = (LongColumnVector) vector; + int curPartLength; + int curPartOffset = 0; + int nextPartLength = size; + while (curPixelIsNullIndex + nextPartLength >= pixelStride) + { + curPartLength = pixelStride - curPixelIsNullIndex; + writeCurPart(columnVector, curPartLength, curPartOffset); + newPixel(); + curPartOffset += curPartLength; + nextPartLength = size - curPartOffset; + } + writeCurPart(columnVector, nextPartLength, curPartOffset); + return outputStream.size(); + } + + private void writeCurPart(LongColumnVector vector, int length, int offset) + { + for (int i = 0; i < length; i++) + { + curPixelEleIndex++; + if (vector.isNull[i + offset]) + { + hasNull = true; + pixelStatRecorder.increment(); + if (nullsPadding) + { + curPixelVector[curPixelVectorIndex++] = 0L; + } + } + else + { + curPixelVector[curPixelVectorIndex++] = vector.vector[i + offset]; + } + } + System.arraycopy(vector.isNull, offset, isNull, curPixelIsNullIndex, length); + curPixelIsNullIndex += length; + } + + @Override + void newPixel() throws IOException + { + if (runlengthEncoding) + { + for (int i = 0; i < curPixelVectorIndex; i++) + { + pixelStatRecorder.updateInteger(curPixelVector[i], 1); + } + outputStream.write(encoder.encode(curPixelVector, 0, curPixelVectorIndex)); + } + else + { + ByteBuffer buffer = ByteBuffer.allocate(curPixelVectorIndex * Long.BYTES).order(byteOrder); + for (int i = 0; i < curPixelVectorIndex; i++) + { + buffer.putLong(curPixelVector[i]); + pixelStatRecorder.updateInteger(curPixelVector[i], 1); + } + outputStream.write(buffer.array()); + } + super.newPixel(); + } + + @Override + public PixelsProto.ColumnEncoding.Builder getColumnChunkEncoding() + { + return PixelsProto.ColumnEncoding.newBuilder().setKind(runlengthEncoding ? + PixelsProto.ColumnEncoding.Kind.RUNLENGTH : PixelsProto.ColumnEncoding.Kind.NONE); + } + + @Override + public void close() throws IOException + { + if (runlengthEncoding) + { + encoder.close(); + } + super.close(); + } + + @Override + public boolean decideNullsPadding(PixelsWriterOption writerOption) + { + return !writerOption.getEncodingLevel().ge(EncodingLevel.EL2) && writerOption.isNullsPadding(); + } +} diff --git a/pixels-core/src/main/java/io/pixelsdb/pixels/core/writer/ShortColumnWriter.java b/pixels-core/src/main/java/io/pixelsdb/pixels/core/writer/ShortColumnWriter.java new file mode 100644 index 0000000000..99d04dfbb6 --- /dev/null +++ b/pixels-core/src/main/java/io/pixelsdb/pixels/core/writer/ShortColumnWriter.java @@ -0,0 +1,141 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the Affero GNU General Public License + * along with Pixels. If not, see + * . + */ +package io.pixelsdb.pixels.core.writer; + +import io.pixelsdb.pixels.core.PixelsProto; +import io.pixelsdb.pixels.core.TypeDescription; +import io.pixelsdb.pixels.core.encoding.EncodingLevel; +import io.pixelsdb.pixels.core.encoding.RunLenIntEncoder; +import io.pixelsdb.pixels.core.vector.ColumnVector; +import io.pixelsdb.pixels.core.vector.ShortColumnVector; + +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * This is the column writer for short (int16) columns. + * + * @author gengdy + * @created 2026-08-07 + */ +public class ShortColumnWriter extends BaseColumnWriter +{ + private final short[] curPixelVector = new short[pixelStride]; + private final boolean runlengthEncoding; + + public ShortColumnWriter(TypeDescription type, PixelsWriterOption writerOption) + { + super(type, writerOption); + runlengthEncoding = encodingLevel.ge(EncodingLevel.EL2); + if (runlengthEncoding) + { + encoder = new RunLenIntEncoder(); + } + } + + @Override + public int write(ColumnVector vector, int size) throws IOException + { + ShortColumnVector columnVector = (ShortColumnVector) vector; + int curPartLength; + int curPartOffset = 0; + int nextPartLength = size; + while (curPixelIsNullIndex + nextPartLength >= pixelStride) + { + curPartLength = pixelStride - curPixelIsNullIndex; + writeCurPart(columnVector, curPartLength, curPartOffset); + newPixel(); + curPartOffset += curPartLength; + nextPartLength = size - curPartOffset; + } + writeCurPart(columnVector, nextPartLength, curPartOffset); + return outputStream.size(); + } + + private void writeCurPart(ShortColumnVector vector, int length, int offset) + { + for (int i = 0; i < length; i++) + { + curPixelEleIndex++; + if (vector.isNull[i + offset]) + { + hasNull = true; + pixelStatRecorder.increment(); + if (nullsPadding) + { + curPixelVector[curPixelVectorIndex++] = 0; + } + } + else + { + curPixelVector[curPixelVectorIndex++] = vector.vector[i + offset]; + } + } + System.arraycopy(vector.isNull, offset, isNull, curPixelIsNullIndex, length); + curPixelIsNullIndex += length; + } + + @Override + void newPixel() throws IOException + { + if (runlengthEncoding) + { + for (int i = 0; i < curPixelVectorIndex; i++) + { + pixelStatRecorder.updateInteger(curPixelVector[i], 1); + } + outputStream.write(encoder.encode(curPixelVector, 0, curPixelVectorIndex)); + } + else + { + ByteBuffer buffer = ByteBuffer.allocate(curPixelVectorIndex * Short.BYTES).order(byteOrder); + for (int i = 0; i < curPixelVectorIndex; i++) + { + buffer.putShort(curPixelVector[i]); + pixelStatRecorder.updateInteger(curPixelVector[i], 1); + } + outputStream.write(buffer.array()); + } + super.newPixel(); + } + + @Override + public PixelsProto.ColumnEncoding.Builder getColumnChunkEncoding() + { + return PixelsProto.ColumnEncoding.newBuilder().setKind(runlengthEncoding ? + PixelsProto.ColumnEncoding.Kind.RUNLENGTH : PixelsProto.ColumnEncoding.Kind.NONE); + } + + @Override + public void close() throws IOException + { + if (runlengthEncoding) + { + encoder.close(); + } + super.close(); + } + + @Override + public boolean decideNullsPadding(PixelsWriterOption writerOption) + { + return !writerOption.getEncodingLevel().ge(EncodingLevel.EL2) && writerOption.isNullsPadding(); + } +} diff --git a/pixels-core/src/test/java/io/pixelsdb/pixels/core/TestColumnVector.java b/pixels-core/src/test/java/io/pixelsdb/pixels/core/TestColumnVector.java index b20acb85ca..95a3c96935 100644 --- a/pixels-core/src/test/java/io/pixelsdb/pixels/core/TestColumnVector.java +++ b/pixels-core/src/test/java/io/pixelsdb/pixels/core/TestColumnVector.java @@ -20,9 +20,10 @@ package io.pixelsdb.pixels.core; import io.pixelsdb.pixels.core.vector.BinaryColumnVector; -import io.pixelsdb.pixels.core.vector.ColumnVector; import io.pixelsdb.pixels.core.vector.DoubleColumnVector; +import io.pixelsdb.pixels.core.vector.IntColumnVector; import io.pixelsdb.pixels.core.vector.LongColumnVector; +import io.pixelsdb.pixels.core.vector.ShortColumnVector; import io.pixelsdb.pixels.core.vector.TimestampColumnVector; import io.pixelsdb.pixels.core.vector.VectorizedRowBatch; import org.junit.Test; @@ -33,6 +34,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; /** @@ -49,78 +51,106 @@ public void testCVSet() for (int i = 0; i < 100; i++) { a.vector[i] = i; + a.isNull[i] = false; } - ColumnVector b = new LongColumnVector(100); + a.setWriteIndex(100); + + LongColumnVector b = new LongColumnVector(100); for (int i = 0; i < 100; i++) { b.addElement(i, a); } + + assertEquals(100, b.getWriteIndex()); + assertTrue(b.noNulls); + for (int i = 0; i < b.getWriteIndex(); i++) + { + assertFalse(b.isNull[i]); + assertEquals(i, b.vector[i]); + } + StringBuilder sb = new StringBuilder(); - for (int i = 0; i < b.getLength(); i++) + for (int i = 0; i < b.getWriteIndex(); i++) { b.stringifyValue(sb, i); - sb.append("\n"); + if (i + 1 < b.getWriteIndex()) + { + sb.append('\n'); + } } - System.out.println(sb.toString()); + String[] lines = sb.toString().split("\n"); + assertEquals(100, lines.length); + assertEquals("0", lines[0]); + assertEquals("99", lines[99]); } @Test public void testDateTimeTypes() { Date date = Date.valueOf("1900-12-31"); - System.out.println(date.getTime()); + assertEquals(Date.valueOf("1900-12-31"), date); + assertTrue(date.getTime() < 0); + Time time = Time.valueOf("23:59:59"); - System.out.println((int)time.getTime()); + assertEquals(Time.valueOf("23:59:59").toString(), time.toString()); + Timestamp timestamp = Timestamp.valueOf("2018-05-07 20:39:20"); - System.out.println(timestamp.getNanos()); + assertEquals(0, timestamp.getNanos()); + assertEquals(Timestamp.valueOf("2018-05-07 20:39:20"), timestamp); + date = new Date(System.currentTimeMillis()); - System.out.println(date.toString()); + assertNotNull(date.toString()); time = new Time(System.currentTimeMillis()); - System.out.println(time.toString()); + assertNotNull(time.toString()); } @Test public void testCVCopyFrom() { int testNum = 1000_000; - String mockSchema = "struct"; - - VectorizedRowBatch srcRowBatch = TypeDescription.fromString(mockSchema).createRowBatch( - testNum, TypeDescription.Mode.NONE); - LongColumnVector src0 = (LongColumnVector) srcRowBatch.cols[0]; - DoubleColumnVector src1 = (DoubleColumnVector) srcRowBatch.cols[1]; - BinaryColumnVector src2 = (BinaryColumnVector) srcRowBatch.cols[2]; - TimestampColumnVector src3 = (TimestampColumnVector) srcRowBatch.cols[3]; - - VectorizedRowBatch dstRowBatch = TypeDescription.fromString(mockSchema).createRowBatch( - testNum, TypeDescription.Mode.NONE); - LongColumnVector dst0 = (LongColumnVector) dstRowBatch.cols[0]; - DoubleColumnVector dst1 = (DoubleColumnVector) dstRowBatch.cols[1]; - BinaryColumnVector dst2 = (BinaryColumnVector) dstRowBatch.cols[2]; - TimestampColumnVector dst3 = (TimestampColumnVector) dstRowBatch.cols[3]; + String mockSchema = "struct"; + + VectorizedRowBatch srcRowBatch = TypeDescription.fromString(mockSchema).createRowBatch(testNum); + ShortColumnVector src0 = (ShortColumnVector) srcRowBatch.cols[0]; + IntColumnVector src1 = (IntColumnVector) srcRowBatch.cols[1]; + LongColumnVector src2 = (LongColumnVector) srcRowBatch.cols[2]; + DoubleColumnVector src3 = (DoubleColumnVector) srcRowBatch.cols[3]; + BinaryColumnVector src4 = (BinaryColumnVector) srcRowBatch.cols[4]; + TimestampColumnVector src5 = (TimestampColumnVector) srcRowBatch.cols[5]; + + VectorizedRowBatch dstRowBatch = TypeDescription.fromString(mockSchema).createRowBatch(testNum); + ShortColumnVector dst0 = (ShortColumnVector) dstRowBatch.cols[0]; + IntColumnVector dst1 = (IntColumnVector) dstRowBatch.cols[1]; + LongColumnVector dst2 = (LongColumnVector) dstRowBatch.cols[2]; + DoubleColumnVector dst3 = (DoubleColumnVector) dstRowBatch.cols[3]; + BinaryColumnVector dst4 = (BinaryColumnVector) dstRowBatch.cols[4]; + TimestampColumnVector dst5 = (TimestampColumnVector) dstRowBatch.cols[5]; for (int i = 0; i < testNum; i++) { - src0.vector[i] = i; + src0.vector[i] = (short) i; src1.vector[i] = i; - src2.setVal(i, String.valueOf(i).getBytes()); - src3.set(i, Timestamp.valueOf("2018-05-07 20:39:20")); + src2.vector[i] = i; + src3.vector[i] = i; + src4.setVal(i, String.valueOf(i).getBytes()); + src5.set(i, Timestamp.valueOf("2018-05-07 20:39:20")); } - long begin = System.nanoTime(); dst0.duplicate(src0); dst1.duplicate(src1); dst2.duplicate(src2); dst3.duplicate(src3); - long end = System.nanoTime(); - System.out.println("Copy cost: " + (end - begin)); + dst4.duplicate(src4); + dst5.duplicate(src5); for (int i = 0; i < testNum; i++) { - assert dst0.vector[i] == i; - assert i * 1.0d == dst1.vector[i]; - assertEquals(String.valueOf(i), dst2.toString(i)); - assertEquals(Timestamp.valueOf("2018-05-07 20:39:20"), dst3.asScratchTimestamp(i)); + assertEquals((short) i, dst0.vector[i]); + assertEquals(i, dst1.vector[i]); + assertEquals(i, dst2.vector[i]); + assertEquals(i * 1.0d, dst3.vector[i], 0); + assertEquals(String.valueOf(i), dst4.toString(i)); + assertEquals(Timestamp.valueOf("2018-05-07 20:39:20"), dst5.asScratchTimestamp(i)); } } @@ -131,30 +161,44 @@ public void testColumnDuplication() VectorizedRowBatch rowBatch = TypeDescription.fromString(mockSchema).createRowBatch(); assertFalse(rowBatch.cols[0].duplicated); - assert rowBatch.cols[0].originVecId == -1; + assertEquals(-1, rowBatch.cols[0].originVecId); assertFalse(rowBatch.cols[1].duplicated); - assert rowBatch.cols[1].originVecId == -1; + assertEquals(-1, rowBatch.cols[1].originVecId); assertFalse(rowBatch.cols[2].duplicated); - assert rowBatch.cols[2].originVecId == -1; + assertEquals(-1, rowBatch.cols[2].originVecId); assertFalse(rowBatch.cols[3].duplicated); - assert rowBatch.cols[3].originVecId == -1; + assertEquals(-1, rowBatch.cols[3].originVecId); assertTrue(rowBatch.cols[4].duplicated); - assert rowBatch.cols[4].originVecId == 0; + assertEquals(0, rowBatch.cols[4].originVecId); assertTrue(rowBatch.cols[5].duplicated); - assert rowBatch.cols[5].originVecId == 1; + assertEquals(1, rowBatch.cols[5].originVecId); assertFalse(rowBatch.cols[6].duplicated); - assert rowBatch.cols[6].originVecId == -1; + assertEquals(-1, rowBatch.cols[6].originVecId); } @Test public void testBytesColumnVector() { - BinaryColumnVector cv = new BinaryColumnVector(); + // Keep capacity large enough to avoid BinaryColumnVector.ensureSize growth; + // the lens-copy fix lives on feature/binaryColumn. + int capacity = 10000; + BinaryColumnVector cv = new BinaryColumnVector(capacity); cv.init(); - cv.ensureSize(1000, false); - for (int i = 0; i < 10000; i++) + assertTrue(cv.getLength() >= capacity); + + for (int i = 0; i < capacity; i++) { cv.add("13333333333333333333333333333334"); //32 bytes } + assertEquals(capacity, cv.getWriteIndex()); + assertTrue(cv.noNulls); + assertEquals(32, cv.lens[0]); + assertEquals(32, cv.lens[capacity - 1]); + assertEquals("13333333333333333333333333333334", cv.toString(0)); + assertEquals("13333333333333333333333333333334", cv.toString(capacity - 1)); + + cv.reset(); + assertEquals(0, cv.getWriteIndex()); + assertTrue(cv.noNulls); } } diff --git a/pixels-core/src/test/java/io/pixelsdb/pixels/core/TestParams.java b/pixels-core/src/test/java/io/pixelsdb/pixels/core/TestParams.java index 384c92fe95..a2b2837b47 100644 --- a/pixels-core/src/test/java/io/pixelsdb/pixels/core/TestParams.java +++ b/pixels-core/src/test/java/io/pixelsdb/pixels/core/TestParams.java @@ -21,19 +21,42 @@ import io.pixelsdb.pixels.core.encoding.EncodingLevel; -public class TestParams { +/** + * Shared constants for pixels-core tests. + */ +public class TestParams +{ + /** + * Placeholder for manual/IT tests that need an external path. + */ public static String filePath = ""; + public static int rowNum = 10; - public final static String schemaStr = "__table"; - public final static long blockSize = 1024; - public final static int pixelStride = 16; + /** + * Shared compact schema for tests that need a multi-column TypeDescription. + */ + public static final String SIMPLE_SCHEMA = + "struct<" + + "a:int," + + "b:float," + + "c:double," + + "d:timestamp," + + "e:boolean," + + "f:date," + + "g:time," + + "h:string," + + "i:decimal(18,2)," + + "j:decimal(38,10)" + + ">"; - public final static int rowGroupSize = 16; + public final static String schemaStr = SIMPLE_SCHEMA; - public final static short blockReplication = 4; + public final static long blockSize = 1024 * 1024; + public final static int pixelStride = 16; + public final static int rowGroupSize = 64 * 1024; + public final static short blockReplication = 1; public final static boolean blockPadding = true; public final static EncodingLevel encodingLevel = EncodingLevel.EL0; - - public final static int compressionBlockSize = 16; + public final static int compressionBlockSize = 1; } diff --git a/pixels-core/src/test/java/io/pixelsdb/pixels/core/TestStreamReaderAndWriter.java b/pixels-core/src/test/java/io/pixelsdb/pixels/core/TestStreamReaderAndWriter.java index d9180711c3..257f24cb62 100644 --- a/pixels-core/src/test/java/io/pixelsdb/pixels/core/TestStreamReaderAndWriter.java +++ b/pixels-core/src/test/java/io/pixelsdb/pixels/core/TestStreamReaderAndWriter.java @@ -39,7 +39,7 @@ public class TestStreamReaderAndWriter static int charMaxSize = 55; static boolean nullsPadding = true; static TypeDescription schema; - static LongColumnVector intColumnVector = new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE); + static IntColumnVector intColumnVector = new IntColumnVector(VectorizedRowBatch.DEFAULT_SIZE); static LongColumnVector longColumnVector = new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE); static DecimalColumnVector decimalColumnVector = new DecimalColumnVector(VectorizedRowBatch.DEFAULT_SIZE, 15, 2); static BinaryColumnVector varCharColumnVector = new BinaryColumnVector(VectorizedRowBatch.DEFAULT_SIZE); @@ -242,15 +242,14 @@ private static PixelsReaderOption isHeaderRight(TypeDescription fileSchema) private static void compareColumn(int colIdx, VectorizedRowBatch rowBatch) { ColumnVector column = rowBatch.cols[colIdx]; - if (column instanceof LongColumnVector && - schema.getChildren().get(colIdx).getCategory() == TypeDescription.Category.INT) + if (column instanceof IntColumnVector) { for (int i = 0; i < VectorizedRowBatch.DEFAULT_SIZE; i++) { assert column.noNulls == intColumnVector.noNulls; assert column.isNull[i] == intColumnVector.isNull[i]; assert !intColumnVector.noNulls && column.isNull[i] || - ((LongColumnVector) column).vector[i] == intColumnVector.vector[i]; + ((IntColumnVector) column).vector[i] == intColumnVector.vector[i]; } } else if (column instanceof LongColumnVector ) { diff --git a/pixels-core/src/test/java/io/pixelsdb/pixels/core/TestTypeDescription.java b/pixels-core/src/test/java/io/pixelsdb/pixels/core/TestTypeDescription.java index 70c54bf825..1256a946ec 100644 --- a/pixels-core/src/test/java/io/pixelsdb/pixels/core/TestTypeDescription.java +++ b/pixels-core/src/test/java/io/pixelsdb/pixels/core/TestTypeDescription.java @@ -25,6 +25,7 @@ import io.pixelsdb.pixels.core.vector.DateColumnVector; import io.pixelsdb.pixels.core.vector.IntColumnVector; import io.pixelsdb.pixels.core.vector.LongColumnVector; +import io.pixelsdb.pixels.core.vector.ShortColumnVector; import io.pixelsdb.pixels.core.vector.TimeColumnVector; import io.pixelsdb.pixels.core.vector.TimestampColumnVector; import org.junit.Test; @@ -114,20 +115,15 @@ public void testConvertSqlAndVectorAgree() byteCol.vector[0] = -12; assertConvert(TypeDescription.createByte(), byteCol, 0, "-12", new byte[]{-12}); - // SHORT/INT support both IntColumnVector and LongColumnVector modes. - IntColumnVector intMode = new IntColumnVector(1); - intMode.vector[0] = -1234; - assertConvert(TypeDescription.createShort(), intMode, 0, "-1234", - ByteBuffer.allocate(Integer.BYTES).putInt(-1234).array()); - assertConvert(TypeDescription.createInt(), intMode, 0, "-1234", - ByteBuffer.allocate(Integer.BYTES).putInt(-1234).array()); + ShortColumnVector shortCol = new ShortColumnVector(1); + shortCol.vector[0] = -1234; + assertConvert(TypeDescription.createShort(), shortCol, 0, "-1234", + ByteBuffer.allocate(Short.BYTES).putShort((short) -1234).array()); - LongColumnVector longMode = new LongColumnVector(1); - longMode.vector[0] = 42; - assertConvert(TypeDescription.createShort(), longMode, 0, "42", - ByteBuffer.allocate(Integer.BYTES).putInt(42).array()); - assertConvert(TypeDescription.createInt(), longMode, 0, "42", - ByteBuffer.allocate(Integer.BYTES).putInt(42).array()); + IntColumnVector intCol = new IntColumnVector(1); + intCol.vector[0] = -1234; + assertConvert(TypeDescription.createInt(), intCol, 0, "-1234", + ByteBuffer.allocate(Integer.BYTES).putInt(-1234).array()); LongColumnVector longCol = new LongColumnVector(1); longCol.vector[0] = Long.MAX_VALUE; diff --git a/pixels-core/src/test/java/io/pixelsdb/pixels/core/reader/TestCharColumnReader.java b/pixels-core/src/test/java/io/pixelsdb/pixels/core/reader/TestCharColumnReader.java new file mode 100644 index 0000000000..7017a09abf --- /dev/null +++ b/pixels-core/src/test/java/io/pixelsdb/pixels/core/reader/TestCharColumnReader.java @@ -0,0 +1,319 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the Affero GNU General Public + * License along with Pixels. If not, see + * . + */ +package io.pixelsdb.pixels.core.reader; + +import io.pixelsdb.pixels.core.PixelsProto; +import io.pixelsdb.pixels.core.TypeDescription; +import io.pixelsdb.pixels.core.encoding.EncodingLevel; +import io.pixelsdb.pixels.core.utils.Bitmap; +import io.pixelsdb.pixels.core.vector.BinaryColumnVector; +import io.pixelsdb.pixels.core.writer.CharColumnWriter; +import io.pixelsdb.pixels.core.writer.PixelsWriterOption; +import org.junit.Test; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +import static org.junit.Assert.assertEquals; + +/** + * Memory round-trip tests for CHAR. CharColumnWriter does not pad with trailing zeros; + * values longer than maxLength are truncated by VarcharColumnWriter before encoding. + * + * @author gengdy + * @create 2026-08-07 + */ +public class TestCharColumnReader +{ + private static final int CHAR_MAX_LENGTH = 8; + + private static TypeDescription charType() + { + return TypeDescription.createChar(CHAR_MAX_LENGTH); + } + + private static BinaryColumnVector createSampleVector(int numRows) + { + BinaryColumnVector vector = new BinaryColumnVector(numRows); + // All non-null values are within CHAR_MAX_LENGTH; include empty and max-length boundary. + vector.add(""); + vector.add("a"); + vector.add("ab"); + vector.add("abcdefg"); + vector.addNull(); + vector.add("abcdefgh"); // exactly maxLength + vector.add("xy"); + vector.add("z"); + vector.addNull(); + vector.add("bound"); + vector.add("555"); + vector.add("565"); + vector.add("234"); + vector.add("675"); + vector.add("235"); + vector.add("32434"); // length 5 + vector.addNull(); + vector.add("6"); + vector.add("7"); + vector.add("maxchar!"); // exactly 8 + vector.add("3434"); + vector.add("end"); + return vector; + } + + private static void assertVectorsEqual(BinaryColumnVector expected, BinaryColumnVector actual, int numRows) + { + assertEquals(expected.noNulls, actual.noNulls); + for (int i = 0; i < numRows; ++i) + { + assertEquals("isNull mismatch at row " + i, expected.isNull[i], actual.isNull[i]); + if (expected.noNulls || !expected.isNull[i]) + { + String e = new String(expected.vector[i], expected.start[i], expected.lens[i]); + String a = new String(actual.vector[i], actual.start[i], actual.lens[i]); + assertEquals("value mismatch at row " + i, e, a); + } + } + } + + private static void assertSelectedRoundTrip(boolean nullsPadding) throws IOException + { + int pixelsStride = 10; + int numRows = 22; + int vectorIndex = 3; + PixelsWriterOption writerOption = new PixelsWriterOption() + .pixelStride(pixelsStride).byteOrder(ByteOrder.LITTLE_ENDIAN) + .encodingLevel(EncodingLevel.EL0).nullsPadding(nullsPadding); + CharColumnWriter columnWriter = new CharColumnWriter(charType(), writerOption); + BinaryColumnVector originVector = createSampleVector(numRows); + columnWriter.write(originVector, numRows); + columnWriter.flush(); + columnWriter.close(); + + byte[] content = columnWriter.getColumnChunkContent(); + PixelsProto.ColumnChunkIndex chunkIndex = columnWriter.getColumnChunkIndex().build(); + PixelsProto.ColumnEncoding encoding = columnWriter.getColumnChunkEncoding().build(); + assertEquals(PixelsProto.ColumnEncoding.Kind.NONE, encoding.getKind()); + + Bitmap selected = new Bitmap(numRows, true); + selected.clear(0); + selected.clear(2); + selected.clear(4); + selected.clear(5); + selected.clear(10); + selected.clear(14); + selected.clear(16); + selected.clear(20); + + CharColumnReader columnReader = new CharColumnReader(charType()); + BinaryColumnVector targetVector = new BinaryColumnVector(vectorIndex + numRows); + columnReader.readSelected(ByteBuffer.wrap(content), encoding, 0, numRows, + pixelsStride, vectorIndex, targetVector, chunkIndex, selected); + columnReader.close(); + + int targetIndex = vectorIndex; + for (int i = 0; i < numRows; ++i) + { + if (selected.get(i)) + { + assertEquals("isNull mismatch at selected src=" + i + " dst=" + targetIndex, + originVector.isNull[i], targetVector.isNull[targetIndex]); + if (!originVector.isNull[i]) + { + String e = new String(originVector.vector[i], originVector.start[i], originVector.lens[i]); + String a = new String(targetVector.vector[targetIndex], + targetVector.start[targetIndex], targetVector.lens[targetIndex]); + assertEquals("value mismatch at selected src=" + i + " dst=" + targetIndex, e, a); + } + targetIndex++; + } + } + } + + @Test + public void testNullsPadding() throws IOException + { + int pixelsStride = 10; + int numRows = 22; + PixelsWriterOption writerOption = new PixelsWriterOption() + .pixelStride(pixelsStride).byteOrder(ByteOrder.LITTLE_ENDIAN) + .encodingLevel(EncodingLevel.EL0).nullsPadding(true); + CharColumnWriter columnWriter = new CharColumnWriter(charType(), writerOption); + BinaryColumnVector sourceVector = createSampleVector(numRows); + columnWriter.write(sourceVector, numRows); + columnWriter.flush(); + columnWriter.close(); + + byte[] content = columnWriter.getColumnChunkContent(); + PixelsProto.ColumnChunkIndex chunkIndex = columnWriter.getColumnChunkIndex().build(); + PixelsProto.ColumnEncoding encoding = columnWriter.getColumnChunkEncoding().build(); + assertEquals(PixelsProto.ColumnEncoding.Kind.NONE, encoding.getKind()); + CharColumnReader columnReader = new CharColumnReader(charType()); + BinaryColumnVector targetVector = new BinaryColumnVector(numRows); + columnReader.read(ByteBuffer.wrap(content), encoding, 0, numRows, + pixelsStride, 0, targetVector, chunkIndex); + columnReader.close(); + + assertVectorsEqual(sourceVector, targetVector, numRows); + } + + @Test + public void testWithoutNullsPadding() throws IOException + { + int pixelsStride = 10; + int numRows = 22; + PixelsWriterOption writerOption = new PixelsWriterOption() + .pixelStride(pixelsStride).byteOrder(ByteOrder.LITTLE_ENDIAN) + .encodingLevel(EncodingLevel.EL0).nullsPadding(false); + CharColumnWriter columnWriter = new CharColumnWriter(charType(), writerOption); + BinaryColumnVector sourceVector = createSampleVector(numRows); + columnWriter.write(sourceVector, numRows); + columnWriter.flush(); + columnWriter.close(); + + byte[] content = columnWriter.getColumnChunkContent(); + PixelsProto.ColumnChunkIndex chunkIndex = columnWriter.getColumnChunkIndex().build(); + PixelsProto.ColumnEncoding encoding = columnWriter.getColumnChunkEncoding().build(); + assertEquals(PixelsProto.ColumnEncoding.Kind.NONE, encoding.getKind()); + CharColumnReader columnReader = new CharColumnReader(charType()); + BinaryColumnVector targetVector = new BinaryColumnVector(numRows); + columnReader.read(ByteBuffer.wrap(content), encoding, 0, numRows, + pixelsStride, 0, targetVector, chunkIndex); + columnReader.close(); + + assertVectorsEqual(sourceVector, targetVector, numRows); + } + + @Test + public void testSelected() throws IOException + { + int pixelsStride = 10; + int numRows = 22; + PixelsWriterOption writerOption = new PixelsWriterOption() + .pixelStride(pixelsStride).byteOrder(ByteOrder.LITTLE_ENDIAN) + .encodingLevel(EncodingLevel.EL0).nullsPadding(true); + CharColumnWriter columnWriter = new CharColumnWriter(charType(), writerOption); + BinaryColumnVector sourceVector = createSampleVector(numRows); + columnWriter.write(sourceVector, numRows); + columnWriter.flush(); + columnWriter.close(); + + byte[] content = columnWriter.getColumnChunkContent(); + PixelsProto.ColumnChunkIndex chunkIndex = columnWriter.getColumnChunkIndex().build(); + PixelsProto.ColumnEncoding encoding = columnWriter.getColumnChunkEncoding().build(); + CharColumnReader columnReader = new CharColumnReader(charType()); + BinaryColumnVector targetVector = new BinaryColumnVector(numRows); + Bitmap selected = new Bitmap(numRows, true); + selected.clear(0); + selected.clear(10); + selected.clear(20); + columnReader.readSelected(ByteBuffer.wrap(content), encoding, 0, numRows, + pixelsStride, 0, targetVector, chunkIndex, selected); + columnReader.close(); + + for (int i = 0, j = 0; i < numRows; ++i) + { + if (i % 10 != 0) + { + assertEquals(sourceVector.isNull[i], targetVector.isNull[j]); + if (sourceVector.noNulls || !sourceVector.isNull[i]) + { + String e = new String(sourceVector.vector[i], sourceVector.start[i], sourceVector.lens[i]); + String a = new String(targetVector.vector[j], targetVector.start[j], targetVector.lens[j]); + assertEquals(e, a); + } + j++; + } + } + } + + @Test + public void testSelectedWithoutNullsPaddingAtNonZeroVectorIndex() throws IOException + { + assertSelectedRoundTrip(false); + } + + @Test + public void testSelectedWithNullsPaddingAtNonZeroVectorIndex() throws IOException + { + assertSelectedRoundTrip(true); + } + + @Test + public void testLargeFragmented() throws IOException + { + int numBatches = 15; + int numRows = 1024; + PixelsWriterOption writerOption = new PixelsWriterOption() + .pixelStride(10000).byteOrder(ByteOrder.LITTLE_ENDIAN) + .encodingLevel(EncodingLevel.EL0).nullsPadding(false); + CharColumnWriter columnWriter = new CharColumnWriter(charType(), writerOption); + + BinaryColumnVector originVector = new BinaryColumnVector(numRows); + for (int j = 0; j < numRows; j++) + { + if (j % 100 == 0) + { + originVector.addNull(); + } + else + { + // Keep within CHAR_MAX_LENGTH. + originVector.add("v" + (j % 10000)); + } + } + + for (int i = 0; i < numBatches; i++) + { + columnWriter.write(originVector, numRows); + } + columnWriter.flush(); + columnWriter.close(); + + byte[] content = columnWriter.getColumnChunkContent(); + PixelsProto.ColumnChunkIndex chunkIndex = columnWriter.getColumnChunkIndex().build(); + PixelsProto.ColumnEncoding encoding = columnWriter.getColumnChunkEncoding().build(); + assertEquals(PixelsProto.ColumnEncoding.Kind.NONE, encoding.getKind()); + int totalRows = numBatches * numRows; + CharColumnReader columnReader = new CharColumnReader(charType()); + BinaryColumnVector targetVector = new BinaryColumnVector(totalRows); + ByteBuffer buffer = ByteBuffer.wrap(content); + columnReader.read(buffer, encoding, 0, 123, + 10000, 0, targetVector, chunkIndex); + columnReader.read(buffer, encoding, 123, 456, + 10000, 123, targetVector, chunkIndex); + columnReader.read(buffer, encoding, 123 + 456, totalRows - 123 - 456, + 10000, 123 + 456, targetVector, chunkIndex); + columnReader.close(); + + for (int i = 0; i < totalRows; i++) + { + int j = i % numRows; + assertEquals(originVector.isNull[j], targetVector.isNull[i]); + if (targetVector.noNulls || !targetVector.isNull[i]) + { + String e = new String(originVector.vector[j], originVector.start[j], originVector.lens[j]); + String a = new String(targetVector.vector[i], targetVector.start[i], targetVector.lens[i]); + assertEquals(e, a); + } + } + } +} diff --git a/pixels-core/src/test/java/io/pixelsdb/pixels/core/reader/TestIntColumnReader.java b/pixels-core/src/test/java/io/pixelsdb/pixels/core/reader/TestIntColumnReader.java index 9aa1b7f19d..8921723268 100644 --- a/pixels-core/src/test/java/io/pixelsdb/pixels/core/reader/TestIntColumnReader.java +++ b/pixels-core/src/test/java/io/pixelsdb/pixels/core/reader/TestIntColumnReader.java @@ -24,8 +24,7 @@ import io.pixelsdb.pixels.core.encoding.EncodingLevel; import io.pixelsdb.pixels.core.utils.Bitmap; import io.pixelsdb.pixels.core.vector.IntColumnVector; -import io.pixelsdb.pixels.core.vector.LongColumnVector; -import io.pixelsdb.pixels.core.writer.IntegerColumnWriter; +import io.pixelsdb.pixels.core.writer.IntColumnWriter; import io.pixelsdb.pixels.core.writer.PixelsWriterOption; import org.junit.Test; @@ -33,69 +32,141 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; +import static org.junit.Assert.assertEquals; + /** - * @author hank + * Memory round-trip tests for INT after the integer-type split. + * + * @author hank, gengdy * @create 2024-12-07 + * @update 2026-08-08 */ public class TestIntColumnReader { - @Test - public void testNullsPadding() throws IOException + private static IntColumnVector createSampleVector(int numRows) + { + IntColumnVector vector = new IntColumnVector(numRows); + vector.add(100); + vector.add(103); + vector.add(106); + vector.add(34); + vector.addNull(); + vector.add(54); + vector.add(55); + vector.add(67); + vector.addNull(); + vector.add(34); + vector.add(555); + vector.add(565); + vector.add(234); + vector.add(675); + vector.add(235); + vector.add(32434); + vector.addNull(); + vector.add(6); + vector.add(7); + vector.add(Integer.MAX_VALUE); + vector.add(3434); + vector.add(Integer.MIN_VALUE); + return vector; + } + + private static void assertVectorsEqual(IntColumnVector expected, IntColumnVector actual, int numRows) + { + assertEquals(expected.noNulls, actual.noNulls); + for (int i = 0; i < numRows; ++i) + { + assertEquals("isNull mismatch at row " + i, expected.isNull[i], actual.isNull[i]); + if (expected.noNulls || !expected.isNull[i]) + { + assertEquals("value mismatch at row " + i, expected.vector[i], actual.vector[i]); + } + } + } + + private static void assertSelectedRoundTrip(EncodingLevel encodingLevel, boolean nullsPadding, + PixelsProto.ColumnEncoding.Kind expectedEncoding) + throws IOException { int pixelsStride = 10; int numRows = 22; + int vectorIndex = 3; PixelsWriterOption writerOption = new PixelsWriterOption() .pixelStride(pixelsStride).byteOrder(ByteOrder.LITTLE_ENDIAN) - .encodingLevel(EncodingLevel.EL0).nullsPadding(true); - IntegerColumnWriter columnWriter = new IntegerColumnWriter( + .encodingLevel(encodingLevel).nullsPadding(nullsPadding); + IntColumnWriter columnWriter = new IntColumnWriter( TypeDescription.createInt(), writerOption); - LongColumnVector longColumnVector = new LongColumnVector(numRows); - longColumnVector.add(100); - longColumnVector.add(103); - longColumnVector.add(106); - longColumnVector.add(34); - longColumnVector.addNull(); - longColumnVector.add(54); - longColumnVector.add(55); - longColumnVector.add(67); - longColumnVector.addNull(); - longColumnVector.add(34); - longColumnVector.add(555); - longColumnVector.add(565); - longColumnVector.add(234); - longColumnVector.add(675); - longColumnVector.add(235); - longColumnVector.add(32434); - longColumnVector.addNull(); - longColumnVector.add(6); - longColumnVector.add(7); - longColumnVector.add(65656565); - longColumnVector.add(3434); - longColumnVector.add(54578); - columnWriter.write(longColumnVector, numRows); + IntColumnVector originVector = createSampleVector(numRows); + columnWriter.write(originVector, numRows); columnWriter.flush(); columnWriter.close(); byte[] content = columnWriter.getColumnChunkContent(); PixelsProto.ColumnChunkIndex chunkIndex = columnWriter.getColumnChunkIndex().build(); PixelsProto.ColumnEncoding encoding = columnWriter.getColumnChunkEncoding().build(); + assertEquals(expectedEncoding, encoding.getKind()); + + Bitmap selected = new Bitmap(numRows, true); + selected.clear(0); + selected.clear(2); + selected.clear(4); + selected.clear(5); + selected.clear(10); + selected.clear(14); + selected.clear(16); + selected.clear(20); + IntColumnReader columnReader = new IntColumnReader(TypeDescription.createInt()); - IntColumnVector intColumnVector = new IntColumnVector(numRows); - columnReader.read(ByteBuffer.wrap(content), encoding, 0, numRows, - pixelsStride, 0, intColumnVector, chunkIndex); + IntColumnVector targetVector = new IntColumnVector(vectorIndex + numRows); + columnReader.readSelected(ByteBuffer.wrap(content), encoding, 0, numRows, + pixelsStride, vectorIndex, targetVector, chunkIndex, selected); columnReader.close(); + int targetIndex = vectorIndex; for (int i = 0; i < numRows; ++i) { - assert intColumnVector.noNulls == longColumnVector.noNulls; - assert intColumnVector.isNull[i] == longColumnVector.isNull[i]; - if (longColumnVector.noNulls || !longColumnVector.isNull[i]) + if (selected.get(i)) { - assert intColumnVector.vector[i] == longColumnVector.vector[i]; + assertEquals("isNull mismatch at selected src=" + i + " dst=" + targetIndex, + originVector.isNull[i], targetVector.isNull[targetIndex]); + if (!originVector.isNull[i]) + { + assertEquals("value mismatch at selected src=" + i + " dst=" + targetIndex, + originVector.vector[i], targetVector.vector[targetIndex]); + } + targetIndex++; } } } + @Test + public void testNullsPadding() throws IOException + { + int pixelsStride = 10; + int numRows = 22; + PixelsWriterOption writerOption = new PixelsWriterOption() + .pixelStride(pixelsStride).byteOrder(ByteOrder.LITTLE_ENDIAN) + .encodingLevel(EncodingLevel.EL0).nullsPadding(true); + IntColumnWriter columnWriter = new IntColumnWriter( + TypeDescription.createInt(), writerOption); + IntColumnVector sourceVector = createSampleVector(numRows); + columnWriter.write(sourceVector, numRows); + columnWriter.flush(); + columnWriter.close(); + + byte[] content = columnWriter.getColumnChunkContent(); + PixelsProto.ColumnChunkIndex chunkIndex = columnWriter.getColumnChunkIndex().build(); + PixelsProto.ColumnEncoding encoding = columnWriter.getColumnChunkEncoding().build(); + assertEquals(PixelsProto.ColumnEncoding.Kind.NONE, encoding.getKind()); + IntColumnReader columnReader = new IntColumnReader(TypeDescription.createInt()); + IntColumnVector targetVector = new IntColumnVector(numRows); + columnReader.read(ByteBuffer.wrap(content), encoding, 0, numRows, + pixelsStride, 0, targetVector, chunkIndex); + columnReader.close(); + + assertVectorsEqual(sourceVector, targetVector, numRows); + } + @Test public void testWithoutNullsPadding() throws IOException { @@ -104,53 +175,74 @@ public void testWithoutNullsPadding() throws IOException PixelsWriterOption writerOption = new PixelsWriterOption() .pixelStride(pixelsStride).byteOrder(ByteOrder.LITTLE_ENDIAN) .encodingLevel(EncodingLevel.EL0).nullsPadding(false); - IntegerColumnWriter columnWriter = new IntegerColumnWriter( + IntColumnWriter columnWriter = new IntColumnWriter( TypeDescription.createInt(), writerOption); - LongColumnVector longColumnVector = new LongColumnVector(numRows); - longColumnVector.add(100); - longColumnVector.add(103); - longColumnVector.add(106); - longColumnVector.add(34); - longColumnVector.addNull(); - longColumnVector.add(54); - longColumnVector.add(55); - longColumnVector.add(67); - longColumnVector.addNull(); - longColumnVector.add(34); - longColumnVector.add(555); - longColumnVector.add(565); - longColumnVector.add(234); - longColumnVector.add(675); - longColumnVector.add(235); - longColumnVector.add(32434); - longColumnVector.addNull(); - longColumnVector.add(6); - longColumnVector.add(7); - longColumnVector.add(65656565); - longColumnVector.add(3434); - longColumnVector.add(54578); - columnWriter.write(longColumnVector, numRows); + IntColumnVector sourceVector = createSampleVector(numRows); + columnWriter.write(sourceVector, numRows); columnWriter.flush(); columnWriter.close(); byte[] content = columnWriter.getColumnChunkContent(); PixelsProto.ColumnChunkIndex chunkIndex = columnWriter.getColumnChunkIndex().build(); PixelsProto.ColumnEncoding encoding = columnWriter.getColumnChunkEncoding().build(); + assertEquals(PixelsProto.ColumnEncoding.Kind.NONE, encoding.getKind()); IntColumnReader columnReader = new IntColumnReader(TypeDescription.createInt()); - IntColumnVector intColumnVector = new IntColumnVector(numRows); + IntColumnVector targetVector = new IntColumnVector(numRows); columnReader.read(ByteBuffer.wrap(content), encoding, 0, numRows, - pixelsStride, 0, intColumnVector, chunkIndex); + pixelsStride, 0, targetVector, chunkIndex); columnReader.close(); - for (int i = 0; i < numRows; ++i) - { - assert intColumnVector.noNulls == longColumnVector.noNulls; - assert intColumnVector.isNull[i] == longColumnVector.isNull[i]; - if (longColumnVector.noNulls || !longColumnVector.isNull[i]) - { - assert intColumnVector.vector[i] == longColumnVector.vector[i]; - } - } + assertVectorsEqual(sourceVector, targetVector, numRows); + } + + @Test + public void testRunLength() throws IOException + { + int pixelsStride = 10; + int numRows = 22; + PixelsWriterOption writerOption = new PixelsWriterOption() + .pixelStride(pixelsStride).byteOrder(ByteOrder.LITTLE_ENDIAN) + .encodingLevel(EncodingLevel.EL2).nullsPadding(false); + IntColumnWriter columnWriter = new IntColumnWriter( + TypeDescription.createInt(), writerOption); + IntColumnVector originVector = new IntColumnVector(numRows); + originVector.add(5); + originVector.add(5); + originVector.add(5); + originVector.add(5); + originVector.addNull(); + originVector.add(5); + originVector.add(5); + originVector.add(7); + originVector.addNull(); + originVector.add(7); + originVector.add(1); + originVector.add(2); + originVector.add(3); + originVector.add(9); + originVector.add(9); + originVector.add(9); + originVector.addNull(); + originVector.add(9); + originVector.add(9); + originVector.add(Integer.MIN_VALUE); + originVector.add(Integer.MAX_VALUE); + originVector.add(0); + columnWriter.write(originVector, numRows); + columnWriter.flush(); + columnWriter.close(); + + byte[] content = columnWriter.getColumnChunkContent(); + PixelsProto.ColumnChunkIndex chunkIndex = columnWriter.getColumnChunkIndex().build(); + PixelsProto.ColumnEncoding encoding = columnWriter.getColumnChunkEncoding().build(); + assertEquals(PixelsProto.ColumnEncoding.Kind.RUNLENGTH, encoding.getKind()); + IntColumnReader columnReader = new IntColumnReader(TypeDescription.createInt()); + IntColumnVector targetVector = new IntColumnVector(numRows); + columnReader.read(ByteBuffer.wrap(content), encoding, 0, numRows, + pixelsStride, 0, targetVector, chunkIndex); + columnReader.close(); + + assertVectorsEqual(originVector, targetVector, numRows); } @Test @@ -161,32 +253,10 @@ public void testSelected() throws IOException PixelsWriterOption writerOption = new PixelsWriterOption() .pixelStride(pixelsStride).byteOrder(ByteOrder.LITTLE_ENDIAN) .encodingLevel(EncodingLevel.EL0).nullsPadding(true); - IntegerColumnWriter columnWriter = new IntegerColumnWriter( + IntColumnWriter columnWriter = new IntColumnWriter( TypeDescription.createInt(), writerOption); - LongColumnVector longColumnVector = new LongColumnVector(numRows); - longColumnVector.add(100); - longColumnVector.add(103); - longColumnVector.add(106); - longColumnVector.add(34); - longColumnVector.addNull(); - longColumnVector.add(54); - longColumnVector.add(55); - longColumnVector.add(67); - longColumnVector.addNull(); - longColumnVector.add(34); - longColumnVector.add(555); - longColumnVector.add(565); - longColumnVector.add(234); - longColumnVector.add(675); - longColumnVector.add(235); - longColumnVector.add(32434); - longColumnVector.addNull(); - longColumnVector.add(6); - longColumnVector.add(7); - longColumnVector.add(65656565); - longColumnVector.add(3434); - longColumnVector.add(54578); - columnWriter.write(longColumnVector, numRows); + IntColumnVector sourceVector = createSampleVector(numRows); + columnWriter.write(sourceVector, numRows); columnWriter.flush(); columnWriter.close(); @@ -194,24 +264,23 @@ public void testSelected() throws IOException PixelsProto.ColumnChunkIndex chunkIndex = columnWriter.getColumnChunkIndex().build(); PixelsProto.ColumnEncoding encoding = columnWriter.getColumnChunkEncoding().build(); IntColumnReader columnReader = new IntColumnReader(TypeDescription.createInt()); - IntColumnVector intColumnVector = new IntColumnVector(numRows); + IntColumnVector targetVector = new IntColumnVector(numRows); Bitmap selected = new Bitmap(numRows, true); selected.clear(0); selected.clear(10); selected.clear(20); columnReader.readSelected(ByteBuffer.wrap(content), encoding, 0, numRows, - pixelsStride, 0, intColumnVector, chunkIndex, selected); + pixelsStride, 0, targetVector, chunkIndex, selected); columnReader.close(); for (int i = 0, j = 0; i < numRows; ++i) { if (i % 10 != 0) { - assert intColumnVector.noNulls == longColumnVector.noNulls; - assert intColumnVector.isNull[j] == longColumnVector.isNull[i]; - if (longColumnVector.noNulls || !longColumnVector.isNull[i]) + assertEquals(sourceVector.isNull[i], targetVector.isNull[j]); + if (sourceVector.noNulls || !sourceVector.isNull[i]) { - assert intColumnVector.vector[j] == longColumnVector.vector[i]; + assertEquals(sourceVector.vector[i], targetVector.vector[j]); } j++; } @@ -219,58 +288,17 @@ public void testSelected() throws IOException } @Test - public void testLarge() throws IOException + public void testSelectedWithoutNullsPaddingAtNonZeroVectorIndex() throws IOException { - int numBatches = 15; - int numRows = 1024; - PixelsWriterOption writerOption = new PixelsWriterOption() - .pixelStride(10000).byteOrder(ByteOrder.LITTLE_ENDIAN) - .encodingLevel(EncodingLevel.EL0).nullsPadding(true); - IntegerColumnWriter columnWriter = new IntegerColumnWriter( - TypeDescription.createInt(), writerOption); - - LongColumnVector originVector = new LongColumnVector(numRows); - for (int j = 0; j < numRows; j++) - { - if (j % 100 == 0) - { - originVector.addNull(); - } - else - { - originVector.add(1000L); - } - } - - for (int i = 0; i < numBatches; i++) - { - columnWriter.write(originVector, numRows); - } - columnWriter.flush(); - columnWriter.close(); - - byte[] content = columnWriter.getColumnChunkContent(); - PixelsProto.ColumnChunkIndex chunkIndex = columnWriter.getColumnChunkIndex().build(); - PixelsProto.ColumnEncoding encoding = columnWriter.getColumnChunkEncoding().build(); - IntColumnReader columnReader = new IntColumnReader(TypeDescription.createInt()); - IntColumnVector targetVector = new IntColumnVector(numBatches*numRows); - columnReader.read(ByteBuffer.wrap(content), encoding, 0, numBatches*numRows, - 10000, 0, targetVector, chunkIndex); - columnReader.close(); + assertSelectedRoundTrip(EncodingLevel.EL0, false, PixelsProto.ColumnEncoding.Kind.NONE); + } - for (int i = 0; i < numBatches*numRows; i++) - { - assert targetVector.isNull[i] == originVector.isNull[i%numRows]; - if (targetVector.noNulls || !targetVector.isNull[i]) - { - assert targetVector.vector[i] == originVector.vector[i % numRows]; - } - } + @Test + public void testSelectedRunLengthAtNonZeroVectorIndex() throws IOException + { + assertSelectedRoundTrip(EncodingLevel.EL2, false, PixelsProto.ColumnEncoding.Kind.RUNLENGTH); } - /** - * Test reading into column vectors with a run-length smaller than pixels stride. - */ @Test public void testLargeFragmented() throws IOException { @@ -278,11 +306,11 @@ public void testLargeFragmented() throws IOException int numRows = 1024; PixelsWriterOption writerOption = new PixelsWriterOption() .pixelStride(10000).byteOrder(ByteOrder.LITTLE_ENDIAN) - .encodingLevel(EncodingLevel.EL0).nullsPadding(true); - IntegerColumnWriter columnWriter = new IntegerColumnWriter( + .encodingLevel(EncodingLevel.EL2).nullsPadding(false); + IntColumnWriter columnWriter = new IntColumnWriter( TypeDescription.createInt(), writerOption); - LongColumnVector originVector = new LongColumnVector(numRows); + IntColumnVector originVector = new IntColumnVector(numRows); for (int j = 0; j < numRows; j++) { if (j % 100 == 0) @@ -291,7 +319,7 @@ public void testLargeFragmented() throws IOException } else { - originVector.add(1000L); + originVector.add((j / 200) % 4); } } @@ -305,23 +333,25 @@ public void testLargeFragmented() throws IOException byte[] content = columnWriter.getColumnChunkContent(); PixelsProto.ColumnChunkIndex chunkIndex = columnWriter.getColumnChunkIndex().build(); PixelsProto.ColumnEncoding encoding = columnWriter.getColumnChunkEncoding().build(); + assertEquals(PixelsProto.ColumnEncoding.Kind.RUNLENGTH, encoding.getKind()); + int totalRows = numBatches * numRows; IntColumnReader columnReader = new IntColumnReader(TypeDescription.createInt()); - IntColumnVector targetVector = new IntColumnVector(numBatches*numRows); + IntColumnVector targetVector = new IntColumnVector(totalRows); ByteBuffer buffer = ByteBuffer.wrap(content); columnReader.read(buffer, encoding, 0, 123, 10000, 0, targetVector, chunkIndex); columnReader.read(buffer, encoding, 123, 456, 10000, 123, targetVector, chunkIndex); - columnReader.read(buffer, encoding, 123+456, numBatches*numRows-123-456, - 10000, 123+456, targetVector, chunkIndex); + columnReader.read(buffer, encoding, 123 + 456, totalRows - 123 - 456, + 10000, 123 + 456, targetVector, chunkIndex); columnReader.close(); - for (int i = 0; i < numBatches*numRows; i++) + for (int i = 0; i < totalRows; i++) { - assert targetVector.isNull[i] == originVector.isNull[i%numRows]; + assertEquals(originVector.isNull[i % numRows], targetVector.isNull[i]); if (targetVector.noNulls || !targetVector.isNull[i]) { - assert targetVector.vector[i] == originVector.vector[i % numRows]; + assertEquals(originVector.vector[i % numRows], targetVector.vector[i]); } } } diff --git a/pixels-core/src/test/java/io/pixelsdb/pixels/core/reader/TestLongColumnReader.java b/pixels-core/src/test/java/io/pixelsdb/pixels/core/reader/TestLongColumnReader.java index ce7798d074..c4a75a6546 100644 --- a/pixels-core/src/test/java/io/pixelsdb/pixels/core/reader/TestLongColumnReader.java +++ b/pixels-core/src/test/java/io/pixelsdb/pixels/core/reader/TestLongColumnReader.java @@ -24,7 +24,7 @@ import io.pixelsdb.pixels.core.encoding.EncodingLevel; import io.pixelsdb.pixels.core.utils.Bitmap; import io.pixelsdb.pixels.core.vector.LongColumnVector; -import io.pixelsdb.pixels.core.writer.IntegerColumnWriter; +import io.pixelsdb.pixels.core.writer.LongColumnWriter; import io.pixelsdb.pixels.core.writer.PixelsWriterOption; import org.junit.Test; @@ -32,124 +32,139 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; +import static org.junit.Assert.assertEquals; + /** - * @author hank + * Memory round-trip tests for LONG after the integer-type split. + * + * @author hank, gengdy * @create 2023-08-21 + * @update 2026-08-08 */ public class TestLongColumnReader { - @Test - public void testNullsPaddingInt() throws IOException + private static LongColumnVector createSampleVector(int numRows) + { + LongColumnVector vector = new LongColumnVector(numRows); + vector.add(100); + vector.add(103); + vector.add(106); + vector.add(34); + vector.addNull(); + vector.add(54); + vector.add(55); + vector.add(67); + vector.addNull(); + vector.add(34); + vector.add(555); + vector.add(565); + vector.add(234); + vector.add(675); + vector.add(235); + vector.add(32434); + vector.addNull(); + vector.add(6); + vector.add(7); + vector.add(Long.MAX_VALUE); + vector.add(3434); + vector.add(Long.MIN_VALUE); + return vector; + } + + private static void assertVectorsEqual(LongColumnVector expected, LongColumnVector actual, int numRows) + { + assertEquals(expected.noNulls, actual.noNulls); + for (int i = 0; i < numRows; ++i) + { + assertEquals("isNull mismatch at row " + i, expected.isNull[i], actual.isNull[i]); + if (expected.noNulls || !expected.isNull[i]) + { + assertEquals("value mismatch at row " + i, expected.vector[i], actual.vector[i]); + } + } + } + + private static void assertSelectedRoundTrip(EncodingLevel encodingLevel, boolean nullsPadding, + PixelsProto.ColumnEncoding.Kind expectedEncoding) + throws IOException { int pixelsStride = 10; int numRows = 22; + int vectorIndex = 3; PixelsWriterOption writerOption = new PixelsWriterOption() .pixelStride(pixelsStride).byteOrder(ByteOrder.LITTLE_ENDIAN) - .encodingLevel(EncodingLevel.EL0).nullsPadding(true); - IntegerColumnWriter columnWriter = new IntegerColumnWriter( - TypeDescription.createInt(), writerOption); - LongColumnVector longColumnVector = new LongColumnVector(numRows); - longColumnVector.add(100); - longColumnVector.add(103); - longColumnVector.add(106); - longColumnVector.add(34); - longColumnVector.addNull(); - longColumnVector.add(54); - longColumnVector.add(55); - longColumnVector.add(67); - longColumnVector.addNull(); - longColumnVector.add(34); - longColumnVector.add(555); - longColumnVector.add(565); - longColumnVector.add(234); - longColumnVector.add(675); - longColumnVector.add(235); - longColumnVector.add(32434); - longColumnVector.addNull(); - longColumnVector.add(6); - longColumnVector.add(7); - longColumnVector.add(65656565); - longColumnVector.add(3434); - longColumnVector.add(54578); - columnWriter.write(longColumnVector, numRows); + .encodingLevel(encodingLevel).nullsPadding(nullsPadding); + LongColumnWriter columnWriter = new LongColumnWriter( + TypeDescription.createLong(), writerOption); + LongColumnVector originVector = createSampleVector(numRows); + columnWriter.write(originVector, numRows); columnWriter.flush(); columnWriter.close(); byte[] content = columnWriter.getColumnChunkContent(); PixelsProto.ColumnChunkIndex chunkIndex = columnWriter.getColumnChunkIndex().build(); PixelsProto.ColumnEncoding encoding = columnWriter.getColumnChunkEncoding().build(); - LongColumnReader columnReader = new LongColumnReader(TypeDescription.createInt()); - LongColumnVector longColumnVector1 = new LongColumnVector(numRows); - columnReader.read(ByteBuffer.wrap(content), encoding, 0, numRows, - pixelsStride, 0, longColumnVector1, chunkIndex); + assertEquals(expectedEncoding, encoding.getKind()); + + Bitmap selected = new Bitmap(numRows, true); + selected.clear(0); + selected.clear(2); + selected.clear(4); + selected.clear(5); + selected.clear(10); + selected.clear(14); + selected.clear(16); + selected.clear(20); + + LongColumnReader columnReader = new LongColumnReader(TypeDescription.createLong()); + LongColumnVector targetVector = new LongColumnVector(vectorIndex + numRows); + columnReader.readSelected(ByteBuffer.wrap(content), encoding, 0, numRows, + pixelsStride, vectorIndex, targetVector, chunkIndex, selected); columnReader.close(); + int targetIndex = vectorIndex; for (int i = 0; i < numRows; ++i) { - assert longColumnVector1.noNulls == longColumnVector.noNulls; - assert longColumnVector1.isNull[i] == longColumnVector.isNull[i]; - if (longColumnVector.noNulls || !longColumnVector.isNull[i]) + if (selected.get(i)) { - assert longColumnVector1.vector[i] == longColumnVector.vector[i]; + assertEquals("isNull mismatch at selected src=" + i + " dst=" + targetIndex, + originVector.isNull[i], targetVector.isNull[targetIndex]); + if (!originVector.isNull[i]) + { + assertEquals("value mismatch at selected src=" + i + " dst=" + targetIndex, + originVector.vector[i], targetVector.vector[targetIndex]); + } + targetIndex++; } } } @Test - public void testNullsPaddingLong() throws IOException + public void testNullsPadding() throws IOException { int pixelsStride = 10; int numRows = 22; PixelsWriterOption writerOption = new PixelsWriterOption() .pixelStride(pixelsStride).byteOrder(ByteOrder.LITTLE_ENDIAN) .encodingLevel(EncodingLevel.EL0).nullsPadding(true); - IntegerColumnWriter columnWriter = new IntegerColumnWriter( + LongColumnWriter columnWriter = new LongColumnWriter( TypeDescription.createLong(), writerOption); - LongColumnVector longColumnVector = new LongColumnVector(numRows); - longColumnVector.add(100); - longColumnVector.add(103); - longColumnVector.add(106); - longColumnVector.add(34); - longColumnVector.addNull(); - longColumnVector.add(54); - longColumnVector.add(55); - longColumnVector.add(67); - longColumnVector.addNull(); - longColumnVector.add(34); - longColumnVector.add(555); - longColumnVector.add(565); - longColumnVector.add(234); - longColumnVector.add(675); - longColumnVector.add(235); - longColumnVector.add(32434); - longColumnVector.addNull(); - longColumnVector.add(6); - longColumnVector.add(7); - longColumnVector.add(65656565); - longColumnVector.add(3434); - longColumnVector.add(54578); - columnWriter.write(longColumnVector, numRows); + LongColumnVector sourceVector = createSampleVector(numRows); + columnWriter.write(sourceVector, numRows); columnWriter.flush(); columnWriter.close(); byte[] content = columnWriter.getColumnChunkContent(); PixelsProto.ColumnChunkIndex chunkIndex = columnWriter.getColumnChunkIndex().build(); PixelsProto.ColumnEncoding encoding = columnWriter.getColumnChunkEncoding().build(); + assertEquals(PixelsProto.ColumnEncoding.Kind.NONE, encoding.getKind()); LongColumnReader columnReader = new LongColumnReader(TypeDescription.createLong()); - LongColumnVector longColumnVector1 = new LongColumnVector(numRows); + LongColumnVector targetVector = new LongColumnVector(numRows); columnReader.read(ByteBuffer.wrap(content), encoding, 0, numRows, - pixelsStride, 0, longColumnVector1, chunkIndex); + pixelsStride, 0, targetVector, chunkIndex); columnReader.close(); - for (int i = 0; i < numRows; ++i) - { - assert longColumnVector1.noNulls == longColumnVector.noNulls; - assert longColumnVector1.isNull[i] == longColumnVector.isNull[i]; - if (longColumnVector.noNulls || !longColumnVector.isNull[i]) - { - assert longColumnVector1.vector[i] == longColumnVector.vector[i]; - } - } + assertVectorsEqual(sourceVector, targetVector, numRows); } @Test @@ -160,148 +175,88 @@ public void testWithoutNullsPadding() throws IOException PixelsWriterOption writerOption = new PixelsWriterOption() .pixelStride(pixelsStride).byteOrder(ByteOrder.LITTLE_ENDIAN) .encodingLevel(EncodingLevel.EL0).nullsPadding(false); - IntegerColumnWriter columnWriter = new IntegerColumnWriter( + LongColumnWriter columnWriter = new LongColumnWriter( TypeDescription.createLong(), writerOption); - LongColumnVector longColumnVector = new LongColumnVector(numRows); - longColumnVector.add(100); - longColumnVector.add(103); - longColumnVector.add(106); - longColumnVector.add(34); - longColumnVector.addNull(); - longColumnVector.add(54); - longColumnVector.add(55); - longColumnVector.add(67); - longColumnVector.addNull(); - longColumnVector.add(34); - longColumnVector.add(555); - longColumnVector.add(565); - longColumnVector.add(234); - longColumnVector.add(675); - longColumnVector.add(235); - longColumnVector.add(32434); - longColumnVector.addNull(); - longColumnVector.add(6); - longColumnVector.add(7); - longColumnVector.add(65656565); - longColumnVector.add(3434); - longColumnVector.add(54578); - columnWriter.write(longColumnVector, numRows); + LongColumnVector sourceVector = createSampleVector(numRows); + columnWriter.write(sourceVector, numRows); columnWriter.flush(); columnWriter.close(); byte[] content = columnWriter.getColumnChunkContent(); PixelsProto.ColumnChunkIndex chunkIndex = columnWriter.getColumnChunkIndex().build(); PixelsProto.ColumnEncoding encoding = columnWriter.getColumnChunkEncoding().build(); + assertEquals(PixelsProto.ColumnEncoding.Kind.NONE, encoding.getKind()); LongColumnReader columnReader = new LongColumnReader(TypeDescription.createLong()); - LongColumnVector longColumnVector1 = new LongColumnVector(numRows); + LongColumnVector targetVector = new LongColumnVector(numRows); columnReader.read(ByteBuffer.wrap(content), encoding, 0, numRows, - pixelsStride, 0, longColumnVector1, chunkIndex); + pixelsStride, 0, targetVector, chunkIndex); columnReader.close(); - for (int i = 0; i < numRows; ++i) - { - assert longColumnVector1.noNulls == longColumnVector.noNulls; - assert longColumnVector1.isNull[i] == longColumnVector.isNull[i]; - if (longColumnVector.noNulls || !longColumnVector.isNull[i]) - { - assert longColumnVector1.vector[i] == longColumnVector.vector[i]; - } - } + assertVectorsEqual(sourceVector, targetVector, numRows); } @Test - public void testSelected() throws IOException + public void testRunLength() throws IOException { int pixelsStride = 10; int numRows = 22; PixelsWriterOption writerOption = new PixelsWriterOption() .pixelStride(pixelsStride).byteOrder(ByteOrder.LITTLE_ENDIAN) - .encodingLevel(EncodingLevel.EL0).nullsPadding(false); - IntegerColumnWriter columnWriter = new IntegerColumnWriter( + .encodingLevel(EncodingLevel.EL2).nullsPadding(false); + LongColumnWriter columnWriter = new LongColumnWriter( TypeDescription.createLong(), writerOption); - LongColumnVector longColumnVector = new LongColumnVector(numRows); - longColumnVector.add(100); - longColumnVector.add(103); - longColumnVector.add(106); - longColumnVector.add(34); - longColumnVector.addNull(); - longColumnVector.add(54); - longColumnVector.add(55); - longColumnVector.add(67); - longColumnVector.addNull(); - longColumnVector.add(34); - longColumnVector.add(555); - longColumnVector.add(565); - longColumnVector.add(234); - longColumnVector.add(675); - longColumnVector.add(235); - longColumnVector.add(32434); - longColumnVector.addNull(); - longColumnVector.add(6); - longColumnVector.add(7); - longColumnVector.add(65656565); - longColumnVector.add(3434); - longColumnVector.add(54578); - columnWriter.write(longColumnVector, numRows); + LongColumnVector originVector = new LongColumnVector(numRows); + originVector.add(5); + originVector.add(5); + originVector.add(5); + originVector.add(5); + originVector.addNull(); + originVector.add(5); + originVector.add(5); + originVector.add(7); + originVector.addNull(); + originVector.add(7); + originVector.add(1); + originVector.add(2); + originVector.add(3); + originVector.add(9); + originVector.add(9); + originVector.add(9); + originVector.addNull(); + originVector.add(9); + originVector.add(9); + originVector.add(Long.MIN_VALUE); + originVector.add(Long.MAX_VALUE); + originVector.add(0); + columnWriter.write(originVector, numRows); columnWriter.flush(); columnWriter.close(); byte[] content = columnWriter.getColumnChunkContent(); PixelsProto.ColumnChunkIndex chunkIndex = columnWriter.getColumnChunkIndex().build(); PixelsProto.ColumnEncoding encoding = columnWriter.getColumnChunkEncoding().build(); + assertEquals(PixelsProto.ColumnEncoding.Kind.RUNLENGTH, encoding.getKind()); LongColumnReader columnReader = new LongColumnReader(TypeDescription.createLong()); - LongColumnVector longColumnVector1 = new LongColumnVector(numRows); - Bitmap selected = new Bitmap(numRows, true); - selected.clear(0); - selected.clear(10); - selected.clear(20); - columnReader.readSelected(ByteBuffer.wrap(content), encoding, 0, numRows, - pixelsStride, 0, longColumnVector1, chunkIndex, selected); + LongColumnVector targetVector = new LongColumnVector(numRows); + columnReader.read(ByteBuffer.wrap(content), encoding, 0, numRows, + pixelsStride, 0, targetVector, chunkIndex); columnReader.close(); - for (int i = 0, j = 0; i < numRows; ++i) - { - if (i % 10 != 0) - { - assert longColumnVector1.noNulls == longColumnVector.noNulls; - assert longColumnVector1.isNull[j] == longColumnVector.isNull[i]; - if (longColumnVector.noNulls || !longColumnVector.isNull[i]) - { - assert longColumnVector1.vector[j] == longColumnVector.vector[i]; - } - j++; - } - } + assertVectorsEqual(originVector, targetVector, numRows); } @Test - public void testLarge() throws IOException + public void testSelected() throws IOException { - int numBatches = 15; - int numRows = 1024; + int pixelsStride = 10; + int numRows = 22; PixelsWriterOption writerOption = new PixelsWriterOption() - .pixelStride(10000).byteOrder(ByteOrder.LITTLE_ENDIAN) + .pixelStride(pixelsStride).byteOrder(ByteOrder.LITTLE_ENDIAN) .encodingLevel(EncodingLevel.EL0).nullsPadding(true); - IntegerColumnWriter columnWriter = new IntegerColumnWriter( + LongColumnWriter columnWriter = new LongColumnWriter( TypeDescription.createLong(), writerOption); - - LongColumnVector originVector = new LongColumnVector(numRows); - for (int j = 0; j < numRows; j++) - { - if (j % 100 == 0) - { - originVector.addNull(); - } - else - { - originVector.add(1000L); - } - } - - for (int i = 0; i < numBatches; i++) - { - columnWriter.write(originVector, numRows); - } + LongColumnVector sourceVector = createSampleVector(numRows); + columnWriter.write(sourceVector, numRows); columnWriter.flush(); columnWriter.close(); @@ -309,24 +264,41 @@ public void testLarge() throws IOException PixelsProto.ColumnChunkIndex chunkIndex = columnWriter.getColumnChunkIndex().build(); PixelsProto.ColumnEncoding encoding = columnWriter.getColumnChunkEncoding().build(); LongColumnReader columnReader = new LongColumnReader(TypeDescription.createLong()); - LongColumnVector targetVector = new LongColumnVector(numBatches*numRows); - columnReader.read(ByteBuffer.wrap(content), encoding, 0, numBatches*numRows, - 10000, 0, targetVector, chunkIndex); + LongColumnVector targetVector = new LongColumnVector(numRows); + Bitmap selected = new Bitmap(numRows, true); + selected.clear(0); + selected.clear(10); + selected.clear(20); + columnReader.readSelected(ByteBuffer.wrap(content), encoding, 0, numRows, + pixelsStride, 0, targetVector, chunkIndex, selected); columnReader.close(); - for (int i = 0; i < numBatches*numRows; i++) + for (int i = 0, j = 0; i < numRows; ++i) { - assert targetVector.isNull[i] == originVector.isNull[i%numRows]; - if (targetVector.noNulls || !targetVector.isNull[i]) + if (i % 10 != 0) { - assert targetVector.vector[i] == originVector.vector[i % numRows]; + assertEquals(sourceVector.isNull[i], targetVector.isNull[j]); + if (sourceVector.noNulls || !sourceVector.isNull[i]) + { + assertEquals(sourceVector.vector[i], targetVector.vector[j]); + } + j++; } } } - /** - * Test reading into column vectors with a run-length smaller than pixels stride. - */ + @Test + public void testSelectedWithoutNullsPaddingAtNonZeroVectorIndex() throws IOException + { + assertSelectedRoundTrip(EncodingLevel.EL0, false, PixelsProto.ColumnEncoding.Kind.NONE); + } + + @Test + public void testSelectedRunLengthAtNonZeroVectorIndex() throws IOException + { + assertSelectedRoundTrip(EncodingLevel.EL2, false, PixelsProto.ColumnEncoding.Kind.RUNLENGTH); + } + @Test public void testLargeFragmented() throws IOException { @@ -334,8 +306,8 @@ public void testLargeFragmented() throws IOException int numRows = 1024; PixelsWriterOption writerOption = new PixelsWriterOption() .pixelStride(10000).byteOrder(ByteOrder.LITTLE_ENDIAN) - .encodingLevel(EncodingLevel.EL0).nullsPadding(true); - IntegerColumnWriter columnWriter = new IntegerColumnWriter( + .encodingLevel(EncodingLevel.EL2).nullsPadding(false); + LongColumnWriter columnWriter = new LongColumnWriter( TypeDescription.createLong(), writerOption); LongColumnVector originVector = new LongColumnVector(numRows); @@ -347,7 +319,7 @@ public void testLargeFragmented() throws IOException } else { - originVector.add(1000L); + originVector.add((j / 200) % 4); } } @@ -361,23 +333,25 @@ public void testLargeFragmented() throws IOException byte[] content = columnWriter.getColumnChunkContent(); PixelsProto.ColumnChunkIndex chunkIndex = columnWriter.getColumnChunkIndex().build(); PixelsProto.ColumnEncoding encoding = columnWriter.getColumnChunkEncoding().build(); + assertEquals(PixelsProto.ColumnEncoding.Kind.RUNLENGTH, encoding.getKind()); + int totalRows = numBatches * numRows; LongColumnReader columnReader = new LongColumnReader(TypeDescription.createLong()); - LongColumnVector targetVector = new LongColumnVector(numBatches*numRows); + LongColumnVector targetVector = new LongColumnVector(totalRows); ByteBuffer buffer = ByteBuffer.wrap(content); columnReader.read(buffer, encoding, 0, 123, 10000, 0, targetVector, chunkIndex); columnReader.read(buffer, encoding, 123, 456, 10000, 123, targetVector, chunkIndex); - columnReader.read(buffer, encoding, 123+456, numBatches*numRows-123-456, - 10000, 123+456, targetVector, chunkIndex); + columnReader.read(buffer, encoding, 123 + 456, totalRows - 123 - 456, + 10000, 123 + 456, targetVector, chunkIndex); columnReader.close(); - for (int i = 0; i < numBatches*numRows; i++) + for (int i = 0; i < totalRows; i++) { - assert targetVector.isNull[i] == originVector.isNull[i%numRows]; + assertEquals(originVector.isNull[i % numRows], targetVector.isNull[i]); if (targetVector.noNulls || !targetVector.isNull[i]) { - assert targetVector.vector[i] == originVector.vector[i % numRows]; + assertEquals(originVector.vector[i % numRows], targetVector.vector[i]); } } } diff --git a/pixels-core/src/test/java/io/pixelsdb/pixels/core/reader/TestPixelsReaderBasic.java b/pixels-core/src/test/java/io/pixelsdb/pixels/core/reader/TestPixelsReaderBasic.java deleted file mode 100644 index 1dc3563dea..0000000000 --- a/pixels-core/src/test/java/io/pixelsdb/pixels/core/reader/TestPixelsReaderBasic.java +++ /dev/null @@ -1,446 +0,0 @@ -/* - * Copyright 2017-2019 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ -package io.pixelsdb.pixels.core.reader; - -import io.pixelsdb.pixels.common.physical.Storage; -import io.pixelsdb.pixels.common.physical.StorageFactory; -import io.pixelsdb.pixels.core.PixelsFooterCache; -import io.pixelsdb.pixels.core.PixelsProto; -import io.pixelsdb.pixels.core.PixelsReader; -import io.pixelsdb.pixels.core.PixelsReaderImpl; -import io.pixelsdb.pixels.core.vector.*; -import org.junit.FixMethodOrder; -import org.junit.Test; -import org.junit.runners.MethodSorters; - -import java.io.IOException; -import java.util.List; -import java.util.Objects; -import java.util.Random; - -import static junit.framework.TestCase.assertTrue; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; - -/** - * pixels reader basic test - * this test is to guarantee basic correctness of the pixels reader - * - * @author guodong - */ -@FixMethodOrder(MethodSorters.NAME_ASCENDING) -public class TestPixelsReaderBasic -{ - private static final boolean DEBUG = true; - private long elementSize = 0; - - @Test - public void testMetadata() - { - String path = "file:///home/hank/Downloads/pixels/20220306043329_1.pxl"; - PixelsReader reader; - try - { - Storage storage = StorageFactory.Instance().getStorage("file"); - reader = PixelsReaderImpl.newBuilder() - .setStorage(storage) - .setPath(path) - .setPixelsFooterCache(new PixelsFooterCache()) - .build(); - List types = reader.getFooter().getTypesList(); - for (PixelsProto.Type type : types) - { - System.out.println(type); - } - System.out.println(reader.getRowGroupStats().size()); - } - catch (IOException e) - { - e.printStackTrace(); - } - - } - - @Test - public void testReadDictionary() - { - String path = "file:///home/hank/20230126155625_0.pxl"; - PixelsReader reader; - try - { - Storage storage = StorageFactory.Instance().getStorage("file"); - reader = PixelsReaderImpl.newBuilder() - .setStorage(storage) - .setPath(path) - .setPixelsFooterCache(new PixelsFooterCache()) - .build(); - PixelsReaderOption option = new PixelsReaderOption(); - option.skipCorruptRecords(true); - option.tolerantSchemaEvolution(true); - option.enableEncodedColumnVector(true); - option.includeCols(new String[]{"o_orderpriority"}); - option.rgRange(0, 1); - option.transId(1); - PixelsRecordReader recordReader = reader.read(option); - VectorizedRowBatch rowBatch = recordReader.readBatch(1000); - DictionaryColumnVector vector = (DictionaryColumnVector) rowBatch.cols[0]; - } - catch (IOException e) - { - e.printStackTrace(); - } - } - - @Test - public void test0SmallNull() - { - String fileName = "test-small-null.pxl"; - int rowNum = 200_000; - Random random = new Random(); - for (int i = 0; i < 10; i++) - { - int batchSize = random.nextInt(rowNum); - System.out.println("row batch size: " + batchSize); - testContent(fileName, batchSize, rowNum, 1528785092538L, true); - } - } - - @Test - public void test1MidNull() - { - String fileName = "test-mid-null.pxl"; - int rowNum = 2_000_000; - Random random = new Random(); - for (int i = 0; i < 10; i++) - { - int batchSize = random.nextInt(rowNum); - System.out.println("row batch size: " + batchSize); - testContent(fileName, batchSize, rowNum, 1528901945696L, true); - } - } - - @Test - public void test2LargeNull() - { - String fileName = "test-large-null.pxl"; - int rowNum = 20_000_000; - Random random = new Random(); - for (int i = 0; i < 10; i++) - { - int batchSize = random.nextInt(200_000); - System.out.println("row batch size: " + batchSize); - testContent(fileName, batchSize, rowNum, 1528902023606L, true); - } - } - - @Test - public void test3Small() - { - String fileName = "test-small.pxl"; - int rowNum = 200_000; - Random random = new Random(); - for (int i = 0; i < 10; i++) - { - int batchSize = random.nextInt(rowNum); - System.out.println("row batch size: " + batchSize); - testContent(fileName, batchSize, rowNum, 1529129883948L, false); - } - } - - @Test - public void test4Mid() - { - String fileName = "test-mid.pxl"; - int rowNum = 2_000_000; - Random random = new Random(); - for (int i = 0; i < 10; i++) - { - int batchSize = random.nextInt(rowNum); - System.out.println("row batch size: " + batchSize); - testContent(fileName, batchSize, rowNum, 1529130997320L, false); - } - } - - private void testContent(String fileName, int batchSize, int rowNum, long time, boolean hasNull) - { - PixelsReaderOption option = new PixelsReaderOption(); - String[] cols = {"a", "b", "c", "d", "e", "z"}; - option.skipCorruptRecords(true); - option.tolerantSchemaEvolution(true); - option.includeCols(cols); - - VectorizedRowBatch rowBatch; - elementSize = 0; - try (PixelsReader pixelsReader = getReader(fileName); - PixelsRecordReader recordReader = pixelsReader.read(option)) - { - while (true) - { - rowBatch = recordReader.readBatch(batchSize); - LongColumnVector acv = (LongColumnVector) rowBatch.cols[0]; - DoubleColumnVector bcv = (DoubleColumnVector) rowBatch.cols[1]; - DoubleColumnVector ccv = (DoubleColumnVector) rowBatch.cols[2]; - TimestampColumnVector dcv = (TimestampColumnVector) rowBatch.cols[3]; - LongColumnVector ecv = (LongColumnVector) rowBatch.cols[4]; - BinaryColumnVector zcv = (BinaryColumnVector) rowBatch.cols[5]; - if (rowBatch.endOfFile) - { - if (hasNull) - { - assertNullCorrect(rowBatch, acv, bcv, ccv, dcv, ecv, zcv, time); - } - else - { - assertCorrect(rowBatch, acv, bcv, ccv, dcv, ecv, zcv, time); - } - break; - } - if (hasNull) - { - assertNullCorrect(rowBatch, acv, bcv, ccv, dcv, ecv, zcv, time); - } - else - { - assertCorrect(rowBatch, acv, bcv, ccv, dcv, ecv, zcv, time); - } - } - assertEquals(rowNum, elementSize); - } - catch (IOException e) - { - e.printStackTrace(); - } - } - - private void assertNullCorrect(VectorizedRowBatch rowBatch, - LongColumnVector acv, - DoubleColumnVector bcv, - DoubleColumnVector ccv, - TimestampColumnVector dcv, - LongColumnVector ecv, - BinaryColumnVector zcv, - long time) - { - for (int i = 0; i < rowBatch.size; i++) - { - if (elementSize % 100 == 0) - { - if (DEBUG) - { - if (!acv.isNull[i]) - { - System.out.println("[a] size: " + elementSize + ", non null"); - } - if (!bcv.isNull[i]) - { - System.out.println("[b] size: " + elementSize + ", non null"); - } - if (!ccv.isNull[i]) - { - System.out.println("[c] size: " + elementSize + ", non null"); - } - if (!dcv.isNull[i]) - { - System.out.println("[d] size: " + elementSize + ", non null"); - } - if (!ecv.isNull[i]) - { - System.out.println("[e] size: " + elementSize + ", non null"); - } - if (!zcv.isNull[i]) - { - System.out.println("[z] size: " + elementSize + ", non null"); - } - } - else - { - assertTrue(acv.isNull[i]); - assertTrue(bcv.isNull[i]); - assertTrue(ccv.isNull[i]); - assertTrue(dcv.isNull[i]); - assertTrue(ecv.isNull[i]); - assertTrue(zcv.isNull[i]); - } - } - else - { - if (DEBUG) - { - if (elementSize != acv.vector[i]) - { - System.out.println("[a] size: " + elementSize - + ", expected: " + elementSize + ", actual: " + acv.vector[i]); - } - if (Float.compare(elementSize * 3.1415f, (float) bcv.vector[i]) != 0) - { - System.out.println("[b] size: " + elementSize - + ", expected: " + elementSize * 3.1415f + ", actual: " + (float) bcv.vector[i]); - } - if (Math.abs(elementSize * 3.14159d - ccv.vector[i]) > 0.000001) - { - System.out.println("[c] size: " + elementSize - + ", expected: " + elementSize * 3.14159d + ", actual: " + ccv.vector[i]); - } - if (dcv.times[i] != time) - { - System.out.println("[d] size: " + elementSize - + ", expected: " + time + ", actual: " + dcv.times[i]); - } - int expectedBool = elementSize > 25 ? 1 : 0; - if (expectedBool != ecv.vector[i]) - { - System.out.println("[e] size: " + elementSize - + ", expected: " + expectedBool + ", actual: " + ecv.vector[i]); - } - String actualStr = new String(zcv.vector[i], zcv.start[i], zcv.lens[i]); - if (!String.valueOf(elementSize).equals(actualStr)) - { - System.out.println("[z] size: " + elementSize - + ", expected: " + String - .valueOf(elementSize) + ", actual: " + actualStr); - } - } - else - { - assertEquals(elementSize, acv.vector[i]); - assertEquals(elementSize * 3.1415f, bcv.vector[i], 0.000001f); - assertEquals(elementSize * 3.14159d, ccv.vector[i], 0.000001d); - assertEquals(time, dcv.times[i]); - assertEquals((elementSize > 25 ? 1 : 0), ecv.vector[i]); - assertEquals(String.valueOf(elementSize), - new String(zcv.vector[i], zcv.start[i], zcv.lens[i])); - } - } - elementSize++; - } - } - - private void assertCorrect(VectorizedRowBatch rowBatch, - LongColumnVector acv, - DoubleColumnVector bcv, - DoubleColumnVector ccv, - TimestampColumnVector dcv, - LongColumnVector ecv, - BinaryColumnVector zcv, - long time) - { - for (int i = 0; i < rowBatch.size; i++) - { - if (DEBUG) - { - if (elementSize != acv.vector[i]) - { - System.out.println("[a] size: " + elementSize - + ", expected: " + elementSize + ", actual: " + acv.vector[i]); - } - if (acv.isNull[i]) - { - System.out.println("[a] size: " + elementSize + ", null"); - } - if (Float.compare(elementSize * 3.1415f, (float) bcv.vector[i]) != 0) - { - System.out.println("[b] size: " + elementSize - + ", expected: " + elementSize * 3.1415f + ", actual: " + (float) bcv.vector[i]); - } - if (bcv.isNull[i]) - { - System.out.println("[b] size: " + elementSize + ", null"); - } - if (Math.abs(elementSize * 3.14159d - ccv.vector[i]) > 0.000001) - { - System.out.println("[c] size: " + elementSize - + ", expected: " + elementSize * 3.14159d + ", actual: " + ccv.vector[i]); - } - if (ccv.isNull[i]) - { - System.out.println("[c] size: " + elementSize + ", null"); - } - if (dcv.times[i] != time) - { - System.out.println("[d] size: " + elementSize - + ", expected: " + time + ", actual: " + dcv.times[i]); - } - if (dcv.isNull[i]) - { - System.out.println("[d] size: " + elementSize + ", null"); - } - int expectedBool = elementSize > 25000 ? 1 : 0; - if (expectedBool != ecv.vector[i]) - { - System.out.println("[e] size: " + elementSize - + ", expected: " + expectedBool + ", actual: " + ecv.vector[i]); - } - if (ecv.isNull[i]) - { - System.out.println("[e] size: " + elementSize + ", null"); - } - String actualStr = new String(zcv.vector[i], zcv.start[i], zcv.lens[i]); - if (!String.valueOf(elementSize).equals(actualStr)) - { - System.out.println("[z] size: " + elementSize - + ", expected: " + String - .valueOf(elementSize) + ", actual: " + actualStr); - } - if (zcv.isNull[i]) - { - System.out.println("[z] size: " + elementSize + ", null"); - } - } - else - { - assertFalse(acv.isNull[i]); - assertEquals(elementSize, acv.vector[i]); - assertFalse(bcv.isNull[i]); - assertEquals(elementSize * 3.1415f, bcv.vector[i], 0.000001d); - assertFalse(ccv.isNull[i]); - assertEquals(elementSize * 3.14159d, ccv.vector[i], 0.000001f); - assertFalse(dcv.isNull[i]); - assertEquals(dcv.times[i], 1528901945696L); - assertFalse(ecv.isNull[i]); - assertEquals((elementSize > 25000 ? 1 : 0), ecv.vector[i]); - assertFalse(zcv.isNull[i]); - assertEquals(String.valueOf(elementSize), - new String(zcv.vector[i], zcv.start[i], zcv.lens[i])); - } - elementSize++; - } - } - - private PixelsReader getReader(String fileName) - { - PixelsReader pixelsReader = null; - String filePath = Objects.requireNonNull( - this.getClass().getClassLoader().getResource("files/" + fileName)).getPath(); - try - { - Storage storage = StorageFactory.Instance().getStorage("hdfs"); - pixelsReader = PixelsReaderImpl.newBuilder() - .setStorage(storage) - .setPath(filePath) - .build(); - } - catch (IOException e) - { - e.printStackTrace(); - } - - return pixelsReader; - } -} \ No newline at end of file diff --git a/pixels-core/src/test/java/io/pixelsdb/pixels/core/reader/TestPixelsReaderOption.java b/pixels-core/src/test/java/io/pixelsdb/pixels/core/reader/TestPixelsReaderOption.java deleted file mode 100644 index e0d08f5447..0000000000 --- a/pixels-core/src/test/java/io/pixelsdb/pixels/core/reader/TestPixelsReaderOption.java +++ /dev/null @@ -1,209 +0,0 @@ -/* - * Copyright 2017-2019 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ -package io.pixelsdb.pixels.core.reader; - -import io.pixelsdb.pixels.common.physical.Storage; -import io.pixelsdb.pixels.common.physical.StorageFactory; -import io.pixelsdb.pixels.core.PixelsReader; -import io.pixelsdb.pixels.core.PixelsReaderImpl; -import io.pixelsdb.pixels.core.vector.*; -import org.junit.Test; - -import java.io.IOException; -import java.util.Objects; - -import static junit.framework.TestCase.assertEquals; -import static junit.framework.TestCase.assertTrue; - -/** - * pixels reader option test - * this test is to guarantee that the pixels reader is able to handle all kinds of options specified by users - * - * @author guodong - */ -public class TestPixelsReaderOption -{ - private int elementSize = 0; - - @Test - public void test0RGRange() - throws IOException - { - // `test-large-null.pxl` is set as the testing file - // this file consists of 6 row groups - // rg0: 5457920 rows - // rg1: 3493888 rows - // rg2: 3374080 rows - // rg3: 3321856 rows - // rg4: 3321856 rows - // rg5: 1030400 rows - String fileName = "test-large-null.pxl"; - PixelsReader pixelsReader = getReader(fileName); - PixelsRecordReader recordReader; - int batchSize = 10000; - - VectorizedRowBatch rowBatch; - PixelsReaderOption option = new PixelsReaderOption(); - String[] cols = {"a", "b", "c", "d", "e", "z"}; - option.skipCorruptRecords(true); - option.tolerantSchemaEvolution(true); - option.includeCols(cols); - - // the whole file - option.rgRange(0, 6); - recordReader = pixelsReader.read(option); - elementSize = 0; - while (true) - { - rowBatch = recordReader.readBatch(batchSize); - if (rowBatch.endOfFile) - { - assertCorrectness(rowBatch, 1528902023606L, 0); - break; - } - assertCorrectness(rowBatch, 1528902023606L, 0); - } - assertEquals(20_000_000, elementSize); - System.out.println("Done with the whole file"); - recordReader.close(); - - // rg0 - option.rgRange(0, 1); - recordReader = pixelsReader.read(option); - elementSize = 0; - while (true) - { - rowBatch = recordReader.readBatch(batchSize); - if (rowBatch.endOfFile) - { - assertCorrectness(rowBatch, 1528902023606L, 0); - break; - } - assertCorrectness(rowBatch, 1528902023606L, 0); - } - assertEquals(5457920, elementSize); - System.out.println("Done with rg0"); - recordReader.close(); - - // rg1, rg2, rg3, rg4 - option.rgRange(1, 4); - recordReader = pixelsReader.read(option); - elementSize = 0; - while (true) - { - rowBatch = recordReader.readBatch(batchSize); - if (rowBatch.endOfFile) - { - assertCorrectness(rowBatch, 1528902023606L, 5457920); - break; - } - assertCorrectness(rowBatch, 1528902023606L, 5457920); - } - assertEquals(13511680, elementSize); - System.out.println("Done with rg1, rg2, rg3 and rg4"); - recordReader.close(); - - // rg4, rg5 - option.rgRange(4, 2); - recordReader = pixelsReader.read(option); - elementSize = 0; - while (true) - { - rowBatch = recordReader.readBatch(batchSize); - if (rowBatch.endOfFile) - { - assertCorrectness(rowBatch, 1528902023606L, 15647744); - break; - } - assertCorrectness(rowBatch, 1528902023606L, 15647744); - } - assertEquals(4352256, elementSize); - System.out.println("Done with rg4 and rg5"); - recordReader.close(); - - pixelsReader.close(); - } - - private void assertCorrectness(VectorizedRowBatch rowBatch, long time, int start) - { - LongColumnVector acv = (LongColumnVector) rowBatch.cols[0]; - DoubleColumnVector bcv = (DoubleColumnVector) rowBatch.cols[1]; - DoubleColumnVector ccv = (DoubleColumnVector) rowBatch.cols[2]; - TimestampColumnVector dcv = (TimestampColumnVector) rowBatch.cols[3]; - LongColumnVector ecv = (LongColumnVector) rowBatch.cols[4]; - BinaryColumnVector zcv = (BinaryColumnVector) rowBatch.cols[5]; - for (int i = 0; i < rowBatch.size; i++) - { - int rowId = elementSize + start; - if (rowId % 100 == 0) - { - assertTrue(acv.isNull[i]); - assertTrue(bcv.isNull[i]); - assertTrue(ccv.isNull[i]); - assertTrue(dcv.isNull[i]); - assertTrue(ecv.isNull[i]); - assertTrue(zcv.isNull[i]); - } - else - { - assertEquals(rowId, acv.vector[i]); - assertEquals(rowId * 3.1415f, bcv.vector[i], 0.000001f); - assertEquals(rowId * 3.14159d, ccv.vector[i], 0.000001d); - assertEquals(time, dcv.times[i]); - assertEquals(rowId > 25 ? 1 : 0, ecv.vector[i]); - assertEquals(String.valueOf(rowId), - new String(zcv.vector[i], zcv.start[i], zcv.lens[i])); - } - elementSize++; - } - } - - private PixelsReader getReader(String fileName) - { - PixelsReader pixelsReader = null; - String filePath = Objects.requireNonNull( - this.getClass().getClassLoader().getResource("files/" + fileName)).getPath(); - try - { - Storage storage = StorageFactory.Instance().getStorage("hdfs"); - pixelsReader = PixelsReaderImpl.newBuilder() - .setStorage(storage) - .setPath(filePath) - .build(); - } - catch (IOException e) - { - e.printStackTrace(); - } - - return pixelsReader; - } - - int[] a; - @Test - public void testArrayDefinition() - { - - for (int i = 0; i < a.length; ++i) - { - System.out.println(a[i]); - } - } -} diff --git a/pixels-core/src/test/java/io/pixelsdb/pixels/core/reader/TestPixelsRecordReaderBufferImpl.java b/pixels-core/src/test/java/io/pixelsdb/pixels/core/reader/TestPixelsRecordReaderBufferImpl.java index d3895834c9..cca9aeea52 100644 --- a/pixels-core/src/test/java/io/pixelsdb/pixels/core/reader/TestPixelsRecordReaderBufferImpl.java +++ b/pixels-core/src/test/java/io/pixelsdb/pixels/core/reader/TestPixelsRecordReaderBufferImpl.java @@ -78,7 +78,6 @@ public void testReadBatch() throws RetinaException, IOException, TransException, option.skipCorruptRecords(true); option.tolerantSchemaEvolution(true); option.enableEncodedColumnVector(true); - option.readIntColumnAsIntVector(true); option.includeCols(includeCols); option.transId(transId); option.transTimestamp(timeStamp); diff --git a/pixels-core/src/test/java/io/pixelsdb/pixels/core/reader/TestShortColumnReader.java b/pixels-core/src/test/java/io/pixelsdb/pixels/core/reader/TestShortColumnReader.java new file mode 100644 index 0000000000..391f2f6215 --- /dev/null +++ b/pixels-core/src/test/java/io/pixelsdb/pixels/core/reader/TestShortColumnReader.java @@ -0,0 +1,356 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public + * License along with Pixels. If not, see + * . + */ +package io.pixelsdb.pixels.core.reader; + +import io.pixelsdb.pixels.core.PixelsProto; +import io.pixelsdb.pixels.core.TypeDescription; +import io.pixelsdb.pixels.core.encoding.EncodingLevel; +import io.pixelsdb.pixels.core.utils.Bitmap; +import io.pixelsdb.pixels.core.vector.ShortColumnVector; +import io.pixelsdb.pixels.core.writer.PixelsWriterOption; +import io.pixelsdb.pixels.core.writer.ShortColumnWriter; +import org.junit.Test; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +import static org.junit.Assert.assertEquals; + +/** + * @author gengdy + * @create 2026-08-07 + */ +public class TestShortColumnReader +{ + private static ShortColumnVector createSampleVector(int numRows) + { + ShortColumnVector vector = new ShortColumnVector(numRows); + vector.add(100); + vector.add(103); + vector.add(106); + vector.add(34); + vector.addNull(); + vector.add(54); + vector.add(55); + vector.add(67); + vector.addNull(); + vector.add(34); + vector.add(555); + vector.add(565); + vector.add(234); + vector.add(675); + vector.add(235); + vector.add(32434); + vector.addNull(); + vector.add(6); + vector.add(7); + vector.add(Short.MAX_VALUE); + vector.add(3434); + vector.add(Short.MIN_VALUE); + return vector; + } + + private static void assertVectorsEqual(ShortColumnVector expected, ShortColumnVector actual, int numRows) + { + assertEquals(expected.noNulls, actual.noNulls); + for (int i = 0; i < numRows; ++i) + { + assertEquals("isNull mismatch at row " + i, expected.isNull[i], actual.isNull[i]); + if (expected.noNulls || !expected.isNull[i]) + { + assertEquals("value mismatch at row " + i, expected.vector[i], actual.vector[i]); + } + } + } + + private static void assertSelectedRoundTrip(EncodingLevel encodingLevel, boolean nullsPadding, + PixelsProto.ColumnEncoding.Kind expectedEncoding) + throws IOException + { + int pixelsStride = 10; + int numRows = 22; + int vectorIndex = 3; + PixelsWriterOption writerOption = new PixelsWriterOption() + .pixelStride(pixelsStride).byteOrder(ByteOrder.LITTLE_ENDIAN) + .encodingLevel(encodingLevel).nullsPadding(nullsPadding); + ShortColumnWriter columnWriter = new ShortColumnWriter( + TypeDescription.createShort(), writerOption); + ShortColumnVector originVector = createSampleVector(numRows); + columnWriter.write(originVector, numRows); + columnWriter.flush(); + columnWriter.close(); + + byte[] content = columnWriter.getColumnChunkContent(); + PixelsProto.ColumnChunkIndex chunkIndex = columnWriter.getColumnChunkIndex().build(); + PixelsProto.ColumnEncoding encoding = columnWriter.getColumnChunkEncoding().build(); + assertEquals(expectedEncoding, encoding.getKind()); + + Bitmap selected = new Bitmap(numRows, true); + // Skip non-null and null rows so the reader must still consume encoded payload. + selected.clear(0); + selected.clear(2); + selected.clear(4); + selected.clear(5); + selected.clear(10); + selected.clear(14); + selected.clear(16); + selected.clear(20); + + ShortColumnReader columnReader = new ShortColumnReader(TypeDescription.createShort()); + ShortColumnVector targetVector = new ShortColumnVector(vectorIndex + numRows); + columnReader.readSelected(ByteBuffer.wrap(content), encoding, 0, numRows, + pixelsStride, vectorIndex, targetVector, chunkIndex, selected); + columnReader.close(); + + int targetIndex = vectorIndex; + for (int i = 0; i < numRows; ++i) + { + if (selected.get(i)) + { + assertEquals("isNull mismatch at selected src=" + i + " dst=" + targetIndex, + originVector.isNull[i], targetVector.isNull[targetIndex]); + if (!originVector.isNull[i]) + { + assertEquals("value mismatch at selected src=" + i + " dst=" + targetIndex, + originVector.vector[i], targetVector.vector[targetIndex]); + } + targetIndex++; + } + } + } + + @Test + public void testNullsPadding() throws IOException + { + int pixelsStride = 10; + int numRows = 22; + PixelsWriterOption writerOption = new PixelsWriterOption() + .pixelStride(pixelsStride).byteOrder(ByteOrder.LITTLE_ENDIAN) + .encodingLevel(EncodingLevel.EL0).nullsPadding(true); + ShortColumnWriter columnWriter = new ShortColumnWriter( + TypeDescription.createShort(), writerOption); + ShortColumnVector sourceVector = createSampleVector(numRows); + columnWriter.write(sourceVector, numRows); + columnWriter.flush(); + columnWriter.close(); + + byte[] content = columnWriter.getColumnChunkContent(); + PixelsProto.ColumnChunkIndex chunkIndex = columnWriter.getColumnChunkIndex().build(); + PixelsProto.ColumnEncoding encoding = columnWriter.getColumnChunkEncoding().build(); + assertEquals(PixelsProto.ColumnEncoding.Kind.NONE, encoding.getKind()); + ShortColumnReader columnReader = new ShortColumnReader(TypeDescription.createShort()); + ShortColumnVector targetVector = new ShortColumnVector(numRows); + columnReader.read(ByteBuffer.wrap(content), encoding, 0, numRows, + pixelsStride, 0, targetVector, chunkIndex); + columnReader.close(); + + assertVectorsEqual(sourceVector, targetVector, numRows); + } + + @Test + public void testWithoutNullsPadding() throws IOException + { + int pixelsStride = 10; + int numRows = 22; + PixelsWriterOption writerOption = new PixelsWriterOption() + .pixelStride(pixelsStride).byteOrder(ByteOrder.LITTLE_ENDIAN) + .encodingLevel(EncodingLevel.EL0).nullsPadding(false); + ShortColumnWriter columnWriter = new ShortColumnWriter( + TypeDescription.createShort(), writerOption); + ShortColumnVector sourceVector = createSampleVector(numRows); + columnWriter.write(sourceVector, numRows); + columnWriter.flush(); + columnWriter.close(); + + byte[] content = columnWriter.getColumnChunkContent(); + PixelsProto.ColumnChunkIndex chunkIndex = columnWriter.getColumnChunkIndex().build(); + PixelsProto.ColumnEncoding encoding = columnWriter.getColumnChunkEncoding().build(); + assertEquals(PixelsProto.ColumnEncoding.Kind.NONE, encoding.getKind()); + ShortColumnReader columnReader = new ShortColumnReader(TypeDescription.createShort()); + ShortColumnVector targetVector = new ShortColumnVector(numRows); + columnReader.read(ByteBuffer.wrap(content), encoding, 0, numRows, + pixelsStride, 0, targetVector, chunkIndex); + columnReader.close(); + + assertVectorsEqual(sourceVector, targetVector, numRows); + } + + @Test + public void testRunLength() throws IOException + { + int pixelsStride = 10; + int numRows = 22; + PixelsWriterOption writerOption = new PixelsWriterOption() + .pixelStride(pixelsStride).byteOrder(ByteOrder.LITTLE_ENDIAN) + .encodingLevel(EncodingLevel.EL2).nullsPadding(false); + ShortColumnWriter columnWriter = new ShortColumnWriter( + TypeDescription.createShort(), writerOption); + ShortColumnVector originVector = new ShortColumnVector(numRows); + originVector.add(5); + originVector.add(5); + originVector.add(5); + originVector.add(5); + originVector.addNull(); + originVector.add(5); + originVector.add(5); + originVector.add(7); + originVector.addNull(); + originVector.add(7); + originVector.add(1); + originVector.add(2); + originVector.add(3); + originVector.add(9); + originVector.add(9); + originVector.add(9); + originVector.addNull(); + originVector.add(9); + originVector.add(9); + originVector.add(Short.MIN_VALUE); + originVector.add(Short.MAX_VALUE); + originVector.add(0); + columnWriter.write(originVector, numRows); + columnWriter.flush(); + columnWriter.close(); + + byte[] content = columnWriter.getColumnChunkContent(); + PixelsProto.ColumnChunkIndex chunkIndex = columnWriter.getColumnChunkIndex().build(); + PixelsProto.ColumnEncoding encoding = columnWriter.getColumnChunkEncoding().build(); + assertEquals(PixelsProto.ColumnEncoding.Kind.RUNLENGTH, encoding.getKind()); + ShortColumnReader columnReader = new ShortColumnReader(TypeDescription.createShort()); + ShortColumnVector targetVector = new ShortColumnVector(numRows); + columnReader.read(ByteBuffer.wrap(content), encoding, 0, numRows, + pixelsStride, 0, targetVector, chunkIndex); + columnReader.close(); + + assertVectorsEqual(originVector, targetVector, numRows); + } + + @Test + public void testSelected() throws IOException + { + int pixelsStride = 10; + int numRows = 22; + PixelsWriterOption writerOption = new PixelsWriterOption() + .pixelStride(pixelsStride).byteOrder(ByteOrder.LITTLE_ENDIAN) + .encodingLevel(EncodingLevel.EL0).nullsPadding(true); + ShortColumnWriter columnWriter = new ShortColumnWriter( + TypeDescription.createShort(), writerOption); + ShortColumnVector sourceVector = createSampleVector(numRows); + columnWriter.write(sourceVector, numRows); + columnWriter.flush(); + columnWriter.close(); + + byte[] content = columnWriter.getColumnChunkContent(); + PixelsProto.ColumnChunkIndex chunkIndex = columnWriter.getColumnChunkIndex().build(); + PixelsProto.ColumnEncoding encoding = columnWriter.getColumnChunkEncoding().build(); + ShortColumnReader columnReader = new ShortColumnReader(TypeDescription.createShort()); + ShortColumnVector targetVector = new ShortColumnVector(numRows); + Bitmap selected = new Bitmap(numRows, true); + selected.clear(0); + selected.clear(10); + selected.clear(20); + columnReader.readSelected(ByteBuffer.wrap(content), encoding, 0, numRows, + pixelsStride, 0, targetVector, chunkIndex, selected); + columnReader.close(); + + for (int i = 0, j = 0; i < numRows; ++i) + { + if (i % 10 != 0) + { + assertEquals(sourceVector.isNull[i], targetVector.isNull[j]); + if (sourceVector.noNulls || !sourceVector.isNull[i]) + { + assertEquals(sourceVector.vector[i], targetVector.vector[j]); + } + j++; + } + } + } + + @Test + public void testSelectedWithoutNullsPaddingAtNonZeroVectorIndex() throws IOException + { + assertSelectedRoundTrip(EncodingLevel.EL0, false, PixelsProto.ColumnEncoding.Kind.NONE); + } + + @Test + public void testSelectedRunLengthAtNonZeroVectorIndex() throws IOException + { + assertSelectedRoundTrip(EncodingLevel.EL2, false, PixelsProto.ColumnEncoding.Kind.RUNLENGTH); + } + + @Test + public void testLargeFragmented() throws IOException + { + int numBatches = 15; + int numRows = 1024; + PixelsWriterOption writerOption = new PixelsWriterOption() + .pixelStride(10000).byteOrder(ByteOrder.LITTLE_ENDIAN) + .encodingLevel(EncodingLevel.EL2).nullsPadding(false); + ShortColumnWriter columnWriter = new ShortColumnWriter( + TypeDescription.createShort(), writerOption); + + ShortColumnVector originVector = new ShortColumnVector(numRows); + for (int j = 0; j < numRows; j++) + { + if (j % 100 == 0) + { + originVector.addNull(); + } + else + { + originVector.add((short) ((j / 200) % 4)); + } + } + + for (int i = 0; i < numBatches; i++) + { + columnWriter.write(originVector, numRows); + } + columnWriter.flush(); + columnWriter.close(); + + byte[] content = columnWriter.getColumnChunkContent(); + PixelsProto.ColumnChunkIndex chunkIndex = columnWriter.getColumnChunkIndex().build(); + PixelsProto.ColumnEncoding encoding = columnWriter.getColumnChunkEncoding().build(); + assertEquals(PixelsProto.ColumnEncoding.Kind.RUNLENGTH, encoding.getKind()); + int totalRows = numBatches * numRows; + ShortColumnReader columnReader = new ShortColumnReader(TypeDescription.createShort()); + ShortColumnVector targetVector = new ShortColumnVector(totalRows); + ByteBuffer buffer = ByteBuffer.wrap(content); + columnReader.read(buffer, encoding, 0, 123, + 10000, 0, targetVector, chunkIndex); + columnReader.read(buffer, encoding, 123, 456, + 10000, 123, targetVector, chunkIndex); + columnReader.read(buffer, encoding, 123 + 456, totalRows - 123 - 456, + 10000, 123 + 456, targetVector, chunkIndex); + columnReader.close(); + + for (int i = 0; i < totalRows; i++) + { + assertEquals(originVector.isNull[i % numRows], targetVector.isNull[i]); + if (targetVector.noNulls || !targetVector.isNull[i]) + { + assertEquals(originVector.vector[i % numRows], targetVector.vector[i]); + } + } + } +} diff --git a/pixels-core/src/test/java/io/pixelsdb/pixels/core/utils/TestRowBatchFlat.java b/pixels-core/src/test/java/io/pixelsdb/pixels/core/utils/TestRowBatchFlat.java index 3723a327c3..f1acfa264d 100644 --- a/pixels-core/src/test/java/io/pixelsdb/pixels/core/utils/TestRowBatchFlat.java +++ b/pixels-core/src/test/java/io/pixelsdb/pixels/core/utils/TestRowBatchFlat.java @@ -45,7 +45,7 @@ public VectorizedRowBatch CreateRowBatch() { String schemaStr = "struct"; TypeDescription schema = TypeDescription.fromString(schemaStr); - VectorizedRowBatch rowBatch = schema.createRowBatch(VectorizedRowBatch.DEFAULT_SIZE, TypeDescription.Mode.CREATE_INT_VECTOR_FOR_INT); + VectorizedRowBatch rowBatch = schema.createRowBatch(VectorizedRowBatch.DEFAULT_SIZE); ByteColumnVector a_ = (ByteColumnVector) rowBatch.cols[0]; // boolean DateColumnVector b_ = (DateColumnVector) rowBatch.cols[1]; // date DecimalColumnVector c_ = (DecimalColumnVector) rowBatch.cols[2]; // decimal diff --git a/pixels-core/src/test/java/io/pixelsdb/pixels/core/writer/TestPixelsWriter.java b/pixels-core/src/test/java/io/pixelsdb/pixels/core/writer/TestPixelsWriter.java deleted file mode 100644 index 9f70138a22..0000000000 --- a/pixels-core/src/test/java/io/pixelsdb/pixels/core/writer/TestPixelsWriter.java +++ /dev/null @@ -1,327 +0,0 @@ -/* - * Copyright 2017-2019 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ -package io.pixelsdb.pixels.core.writer; - -import io.pixelsdb.pixels.common.physical.Storage; -import io.pixelsdb.pixels.common.physical.StorageFactory; -import io.pixelsdb.pixels.core.*; -import io.pixelsdb.pixels.core.exception.PixelsWriterException; -import io.pixelsdb.pixels.core.reader.PixelsReaderOption; -import io.pixelsdb.pixels.core.reader.PixelsRecordReader; -import io.pixelsdb.pixels.core.vector.*; -import org.junit.Test; - -import java.io.IOException; -import java.sql.Date; -import java.sql.Time; -import java.sql.Timestamp; -import java.util.ArrayList; - -/** - * pixels - * - * @author guodong - * @author hank - */ -public class TestPixelsWriter -{ - - @Test - public void testWriterWithNull() - { - String filePath = TestParams.filePath; - - // schema: struct - try - { - Storage storage = StorageFactory.Instance().getStorage("file"); - TypeDescription schema = TypeDescription.fromString(TestParams.schemaStr); - VectorizedRowBatch rowBatch = schema.createRowBatch(); - LongColumnVector va = (LongColumnVector) rowBatch.cols[0]; // int - DoubleColumnVector vb = (DoubleColumnVector) rowBatch.cols[1]; // float - DoubleColumnVector vc = (DoubleColumnVector) rowBatch.cols[2]; // double - TimestampColumnVector vd = (TimestampColumnVector) rowBatch.cols[3]; // timestamp - ByteColumnVector ve = (ByteColumnVector) rowBatch.cols[4]; // boolean - DateColumnVector vf = (DateColumnVector) rowBatch.cols[5]; // date - TimeColumnVector vg = (TimeColumnVector) rowBatch.cols[6]; // time - BinaryColumnVector vh = (BinaryColumnVector) rowBatch.cols[7]; // string - DecimalColumnVector vi = (DecimalColumnVector) rowBatch.cols[8]; // decimal - LongDecimalColumnVector vj = (LongDecimalColumnVector) rowBatch.cols[9];// long decimal - - System.out.println(vi.getPrecision()); - System.out.println(vi.getScale()); - System.out.println(vj.getPrecision()); - System.out.println(vj.getScale()); - - PixelsWriter pixelsWriter = - PixelsWriterImpl.newBuilder() - .setSchema(schema) - .setPixelStride(TestParams.pixelStride) - .setRowGroupSize(TestParams.rowGroupSize) - .setStorage(storage) - .setPath(filePath) - .setBlockSize(TestParams.blockSize) - .setReplication(TestParams.blockReplication) - .setBlockPadding(TestParams.blockPadding) - .setEncodingLevel(TestParams.encodingLevel) - .setCompressionBlockSize(TestParams.compressionBlockSize) - .build(); - - long curT = System.currentTimeMillis(); - Timestamp timestamp = new Timestamp(curT); - for (int i = 0; i < TestParams.rowNum; i++) - { - int row = rowBatch.size++; - if (i % 100 == 0) - { - va.isNull[row] = true; - va.vector[row] = 0; - vb.isNull[row] = true; - vb.vector[row] = 0; - vc.isNull[row] = true; - vc.vector[row] = 0; - vd.isNull[row] = true; - vd.times[row] = 0; - ve.isNull[row] = true; - ve.vector[row] = 0; - vf.isNull[row] = true; - vf.dates[row] = 0; - vg.isNull[row] = true; - vg.times[row] = 0; - vh.isNull[row] = true; - vh.vector[row] = new byte[0]; - vi.isNull[row] = true; - vi.vector[row] = 0; - vj.isNull[row] = true; - vj.vector[row*2] = 0; - vj.vector[row*2+1] = 0; - } - else - { - va.vector[row] = i; - va.isNull[row] = false; - vb.vector[row] = Float.floatToIntBits(i * 3.1415f); - vb.isNull[row] = false; - vc.vector[row] = Double.doubleToLongBits(i * 3.14159d); - vc.isNull[row] = false; - vd.set(row, timestamp); - vd.isNull[row] = false; - ve.vector[row] = (byte) (i % 100 > 25 ? 1 : 0); - ve.isNull[row] = false; - vf.set(row, new Date(System.currentTimeMillis())); - vf.isNull[row] = false; - vg.set(row, new Time(System.currentTimeMillis())); - vg.isNull[row] = false; - vh.setVal(row, String.valueOf(i).getBytes()); - vh.isNull[row] = false; - vi.vector[row] = i; - vi.isNull[row] = false; - vj.vector[row*2] = i; - vj.vector[row*2+1] = i; - vj.isNull[row] = false; - } - if (rowBatch.size == rowBatch.getMaxSize()) - { - pixelsWriter.addRowBatch(rowBatch); - rowBatch.reset(); - } - } - if (rowBatch.size != 0) - { - pixelsWriter.addRowBatch(rowBatch); - rowBatch.reset(); - } - pixelsWriter.close(); - } - catch (IOException | PixelsWriterException e) - { - e.printStackTrace(); - } - } - - @Test - public void testWriterWithoutNull() - { - String filePath = TestParams.filePath; - - // schema: struct - try - { - Storage storage = StorageFactory.Instance().getStorage("hdfs"); - TypeDescription schema = TypeDescription.fromString(TestParams.schemaStr); - VectorizedRowBatch rowBatch = schema.createRowBatch(); - LongColumnVector a = (LongColumnVector) rowBatch.cols[0]; // int - DoubleColumnVector b = (DoubleColumnVector) rowBatch.cols[1]; // float - DoubleColumnVector c = (DoubleColumnVector) rowBatch.cols[2]; // double - TimestampColumnVector d = (TimestampColumnVector) rowBatch.cols[3]; // timestamp - LongColumnVector e = (LongColumnVector) rowBatch.cols[4]; // boolean - BinaryColumnVector z = (BinaryColumnVector) rowBatch.cols[5]; // string - - PixelsWriter pixelsWriter = - PixelsWriterImpl.newBuilder() - .setSchema(schema) - .setPixelStride(TestParams.pixelStride) - .setRowGroupSize(TestParams.rowGroupSize) - .setStorage(storage) - .setPath(filePath) - .setBlockSize(TestParams.blockSize) - .setReplication(TestParams.blockReplication) - .setBlockPadding(TestParams.blockPadding) - .setEncodingLevel(TestParams.encodingLevel) - .setCompressionBlockSize(TestParams.compressionBlockSize) - .build(); - - long curT = System.currentTimeMillis(); - Timestamp timestamp = new Timestamp(curT); - for (int i = 0; i < TestParams.rowNum; i++) - { - int row = rowBatch.size++; - a.vector[row] = i; - a.isNull[row] = false; - b.vector[row] = Float.floatToIntBits(i * 3.1415f); - b.isNull[row] = false; - c.vector[row] = Double.doubleToLongBits(i * 3.14159d); - c.isNull[row] = false; - d.set(row, timestamp); - d.isNull[row] = false; - e.vector[row] = i > 25000 ? 1 : 0; - e.isNull[row] = false; - z.setVal(row, String.valueOf(i).getBytes()); - z.isNull[row] = false; - if (rowBatch.size == rowBatch.getMaxSize()) - { - pixelsWriter.addRowBatch(rowBatch); - rowBatch.reset(); - } - } - if (rowBatch.size != 0) - { - pixelsWriter.addRowBatch(rowBatch); - rowBatch.reset(); - } - pixelsWriter.close(); - } - catch (IOException | PixelsWriterException e) - { - e.printStackTrace(); - } - } - - @Test - public void testRead() - { - PixelsReaderOption option = new PixelsReaderOption(); - String[] cols = {"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"}; - option.skipCorruptRecords(true); - option.tolerantSchemaEvolution(true); - option.includeCols(cols); - option.rgRange(0, 1); - - VectorizedRowBatch rowBatch; - PixelsReader pixelsReader; - - try - { - Storage storage = StorageFactory.Instance().getStorage("file"); - String path = TestParams.filePath; - pixelsReader = PixelsReaderImpl.newBuilder() - .setStorage(storage) - .setPath(path) - .setEnableCache(false) - .setCacheOrder(new ArrayList<>()) - .setPixelsCacheReader(null) - .setPixelsFooterCache(new PixelsFooterCache()) - .build(); - PixelsRecordReader recordReader = pixelsReader.read(option); - rowBatch = recordReader.readBatch(); - LongColumnVector acv = (LongColumnVector) rowBatch.cols[0]; - DoubleColumnVector bcv = (DoubleColumnVector) rowBatch.cols[1]; - DoubleColumnVector ccv = (DoubleColumnVector) rowBatch.cols[2]; - TimestampColumnVector dcv = (TimestampColumnVector) rowBatch.cols[3]; - ByteColumnVector ecv = (ByteColumnVector) rowBatch.cols[4]; - DateColumnVector fcv = (DateColumnVector) rowBatch.cols[5]; - TimeColumnVector gcv = (TimeColumnVector) rowBatch.cols[6]; - BinaryColumnVector hcv = (BinaryColumnVector) rowBatch.cols[7]; - DecimalColumnVector icv = (DecimalColumnVector) rowBatch.cols[8]; - LongDecimalColumnVector jcv = (LongDecimalColumnVector) rowBatch.cols[9]; - for (int i = 0; i < rowBatch.size; ++i) - { - if (dcv.isNull[i]) - { - System.out.println("null"); - } - else - { - System.out.println(dcv.asScratchTimestamp(i)); - - } - } - } - catch (IOException e) - { - e.printStackTrace(); - } - } - - @Test - public void testReadTpchNation() - { - PixelsReaderOption option = new PixelsReaderOption(); - String[] cols = {"n_nationkey", "n_name", "n_regionkey", "n_comment"}; - option.skipCorruptRecords(true); - option.tolerantSchemaEvolution(true); - option.includeCols(cols); - option.rgRange(0, 1); - - VectorizedRowBatch rowBatch; - PixelsReader pixelsReader; - - try - { - Storage storage = StorageFactory.Instance().getStorage("file"); - String path = TestParams.filePath; - pixelsReader = PixelsReaderImpl.newBuilder() - .setStorage(storage) - .setPath(path) - .setEnableCache(false) - .setCacheOrder(new ArrayList<>()) - .setPixelsCacheReader(null) - .setPixelsFooterCache(new PixelsFooterCache()) - .build(); - PixelsRecordReader recordReader = pixelsReader.read(option); - rowBatch = recordReader.readBatch(); - LongColumnVector nationKeyVector = (LongColumnVector) rowBatch.cols[0]; - BinaryColumnVector nameVector = (BinaryColumnVector) rowBatch.cols[1]; - LongColumnVector regionKeyVector = (LongColumnVector) rowBatch.cols[2]; - BinaryColumnVector commentVector = (BinaryColumnVector) rowBatch.cols[3]; - for (int i = 0; i < rowBatch.size; ++i) - { - String name = new String(nameVector.vector[i], nameVector.start[i], nameVector.lens[i]); - String comment = new String(commentVector.vector[i], commentVector.start[i], commentVector.lens[i]); - System.out.println(nationKeyVector.vector[i] + ", " + name + ", " + regionKeyVector.vector[i] + ", " + comment); - } - pixelsReader.close(); - } - catch (IOException e) - { - e.printStackTrace(); - } - } -} diff --git a/pixels-example/src/main/java/io/pixelsdb/pixels/example/core/TestColumnReader.java b/pixels-example/src/main/java/io/pixelsdb/pixels/example/core/TestColumnReader.java deleted file mode 100644 index 60c8eee6f6..0000000000 --- a/pixels-example/src/main/java/io/pixelsdb/pixels/example/core/TestColumnReader.java +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Copyright 2024 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ -package io.pixelsdb.pixels.example.core; - -import io.pixelsdb.pixels.common.physical.Storage; -import io.pixelsdb.pixels.common.physical.StorageFactory; -import io.pixelsdb.pixels.core.*; -import io.pixelsdb.pixels.core.encoding.EncodingLevel; -import io.pixelsdb.pixels.core.exception.PixelsWriterException; -import io.pixelsdb.pixels.core.reader.PixelsReaderOption; -import io.pixelsdb.pixels.core.reader.PixelsRecordReader; -import io.pixelsdb.pixels.core.vector.*; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; - -import static io.pixelsdb.pixels.core.predicate.PixelsPredicate.TRUE_PREDICATE; - -public class TestColumnReader -{ - public static void main(String[] args) throws IOException - { - String pixelsFile = "/home/pixels/data/tpch_5g/test/test.pxl"; - Storage storage = StorageFactory.Instance().getStorage("file"); - String schemaStr = "struct"; - - try - { - // delete pixel file - Files.deleteIfExists(Paths.get(pixelsFile)); - - // write pixel file - TypeDescription schema = TypeDescription.fromString(schemaStr); - VectorizedRowBatch rowBatch = schema.createRowBatchWithHiddenColumn(); - ByteColumnVector a = (ByteColumnVector) rowBatch.cols[0]; // boolean - DateColumnVector b = (DateColumnVector) rowBatch.cols[1]; // date - DecimalColumnVector c = (DecimalColumnVector) rowBatch.cols[2]; // decimal - DoubleColumnVector d = (DoubleColumnVector) rowBatch.cols[3]; // double - FloatColumnVector e = (FloatColumnVector) rowBatch.cols[4]; // float - LongColumnVector f = (LongColumnVector) rowBatch.cols[5]; // int - LongDecimalColumnVector g = (LongDecimalColumnVector) rowBatch.cols[6]; // long decimal - BinaryColumnVector h = (BinaryColumnVector) rowBatch.cols[7]; // string - TimeColumnVector m = (TimeColumnVector) rowBatch.cols[8]; // time - TimestampColumnVector j = (TimestampColumnVector) rowBatch.cols[9]; // timestamp - VectorColumnVector k = (VectorColumnVector) rowBatch.cols[10]; // vector - LongColumnVector l = (LongColumnVector) rowBatch.cols[11]; // long - - PixelsWriter pixelsWriter = - PixelsWriterImpl.newBuilder() - .setSchema(schema) - .setHasHiddenColumn(true) - .setPixelStride(10000) - .setRowGroupSize(64 * 1024 * 1024) - .setStorage(storage) - .setPath(pixelsFile) - .setBlockSize(256 * 1024 * 1024) - .setReplication((short) 3) - .setBlockPadding(true) - .setEncodingLevel(EncodingLevel.EL2) - .setCompressionBlockSize(1) - .setNullsPadding(true) - .build(); - - for (int i = 0; i < 10; i++) - { - int row = rowBatch.size++; - a.vector[row] = (byte) (i % 2); - a.isNull[row] = false; - b.isNull[row] = true; - c.vector[row] = 10000 + i; - c.isNull[row] = false; - d.isNull[row] = true; - e.vector[row] = 10000 + i; - e.isNull[row] = false; - f.isNull[row] = true; - g.vector[row << 1] = 0; - g.vector[(row << 1) + 1] = 10000 + i; - g.isNull[row] = false; - h.isNull[row] = true; - m.set(row, 1000 * i); - m.isNull[row] = false; - j.isNull[row] = true; - k.setRef(row, new double[]{i + 0.1, i + 0.2}); - k.isNull[row] = false; - l.vector[row] = 100 - i; - l.isNull[row] = false; - } - for (int i = 10; i < 20; i++) - { - int row = rowBatch.size++; - a.isNull[row] = true; - b.dates[row] = 1000 + i; - b.isNull[row] = false; - c.isNull[row] = true; - d.vector[row] = 1000 + i; - d.isNull[row] = false; - e.isNull[row] = true; - f.vector[row] = 1000 + i; - f.isNull[row] = false; - g.isNull[row] = true; - h.setVal(row, String.valueOf(i).getBytes()); - h.isNull[row] = false; - m.isNull[row] = true; - j.set(row, 1000 * i); - j.isNull[row] = false; -// k.setRef(row, new double[]{i + 0.1, i + 0.2}); -// k.isNull[row] = false; - k.isNull[row] = true; - l.vector[row] = 100 - i; - l.isNull[row] = false; - } - for (int i = 20; i < 30; i++) - { - int row = rowBatch.size++; - a.vector[row] = (byte) 1; - a.isNull[row] = false; - b.dates[row] = 1000 + i; - b.isNull[row] = false; - c.vector[row] = 10000 + i; - c.isNull[row] = false; - d.vector[row] = 1000 + i; - d.isNull[row] = false; - e.vector[row] = 10000 + i; - e.isNull[row] = false; - f.vector[row] = 1000 + i; - f.isNull[row] = false; - g.vector[row << 1] = 0; - g.vector[(row << 1) + 1] = 10000 + i; - g.isNull[row] = false; - h.setVal(row, String.valueOf(i).getBytes()); - h.isNull[row] = false; - m.set(row, 1000 * i); - m.isNull[row] = false; - j.set(row, 1000 * i); - j.isNull[row] = false; - k.setRef(row, new double[]{i + 0.1, i + 0.2}); - k.isNull[row] = false; - l.vector[row] = 100 - i; - l.isNull[row] = false; - } - if (rowBatch.size != 0) - { - pixelsWriter.addRowBatch(rowBatch); - System.out.println("A rowBatch of size " + rowBatch.size + " has been written to " + pixelsFile); - rowBatch.reset(); - } - pixelsWriter.close(); - - // read pixel file - PixelsReader reader = PixelsReaderImpl.newBuilder() - .setStorage(storage) - .setPath(pixelsFile) - .setPixelsFooterCache(new PixelsFooterCache()) - .build(); - String[] cols = new String[11]; - cols[0] = reader.getFileSchema().getFieldNames().get(0); - cols[1] = reader.getFileSchema().getFieldNames().get(1); - cols[2] = reader.getFileSchema().getFieldNames().get(2); - cols[3] = reader.getFileSchema().getFieldNames().get(3); - cols[4] = reader.getFileSchema().getFieldNames().get(4); - cols[5] = reader.getFileSchema().getFieldNames().get(5); - cols[6] = reader.getFileSchema().getFieldNames().get(6); - cols[7] = reader.getFileSchema().getFieldNames().get(7); - cols[8] = reader.getFileSchema().getFieldNames().get(8); - cols[9] = reader.getFileSchema().getFieldNames().get(9); - cols[10] = reader.getFileSchema().getFieldNames().get(10); - PixelsReaderOption option = new PixelsReaderOption(); - option.transId(0); - option.transTimestamp(85); - option.skipCorruptRecords(true); - option.tolerantSchemaEvolution(true); - option.includeCols(cols); - option.predicate(TRUE_PREDICATE); - PixelsRecordReader recordReader = reader.read(option); - int batchSize = 11; - VectorizedRowBatch resultBatch; - int len = 0; - int numRows = 0; - int numBatches = 0; - while (true) { - resultBatch = recordReader.readBatch(batchSize); - System.out.println("rowBatch: " + resultBatch); - numBatches++; - String result = resultBatch.toString(); - len += result.length(); - System.out.println("loop:" + numBatches + ", rowBatchSize:" + resultBatch.size); - if (resultBatch.endOfFile) { - numRows += resultBatch.size; - break; - } - numRows += resultBatch.size; - } - reader.close(); - } catch (IOException | PixelsWriterException e) - { - e.printStackTrace(); - } - - // delete pixel file - Files.deleteIfExists(Paths.get(pixelsFile)); - } -} diff --git a/pixels-example/src/main/java/io/pixelsdb/pixels/example/core/TestHiddenColumnReader.java b/pixels-example/src/main/java/io/pixelsdb/pixels/example/core/TestHiddenColumnReader.java new file mode 100644 index 0000000000..630c55dcb7 --- /dev/null +++ b/pixels-example/src/main/java/io/pixelsdb/pixels/example/core/TestHiddenColumnReader.java @@ -0,0 +1,609 @@ +/* + * Copyright 2018 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the Affero GNU General Public + * License along with Pixels. If not, see + * . + */ +package io.pixelsdb.pixels.example.core; + +import io.pixelsdb.pixels.common.physical.Storage; +import io.pixelsdb.pixels.common.physical.StorageFactory; +import io.pixelsdb.pixels.core.*; +import io.pixelsdb.pixels.core.encoding.EncodingLevel; +import io.pixelsdb.pixels.core.reader.PixelsReaderOption; +import io.pixelsdb.pixels.core.reader.PixelsRecordReader; +import io.pixelsdb.pixels.core.utils.Bitmap; +import io.pixelsdb.pixels.core.vector.IntColumnVector; +import io.pixelsdb.pixels.core.vector.LongColumnVector; +import io.pixelsdb.pixels.core.vector.VectorizedRowBatch; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; + +/** + * Tests for hidden-timestamp-column expose/filter logic in PixelsRecordReaderImpl. + * + * Covers: + * T1 empty projection + expose only (no filter) + * T2 empty projection + filter + expose + * T3 with projection + filter + expose + * T4 with projection + expose only (no filter) + * T5 multi-RG stride correctness (small batch across RG boundaries) + * T6 no hidden column + valid transTimestamp (must not crash) + * T7 empty projection + filter + no expose (count(*) with timestamp filter) + * T8 expose + no-hidden-column file (must throw IOException) + * T9 filter + no expose (hiddenColumnVector must be null) + * T10 VectorizedRowBatch.applyFilter syncs hiddenColumnVector + */ +public class TestHiddenColumnReader +{ + private static final String TEST_DIR = "/tmp/pixels_test_reader/"; + private static final String FILE_WITH_HIDDEN = TEST_DIR + "with_hidden.pxl"; + private static final String FILE_NO_HIDDEN = TEST_DIR + "no_hidden.pxl"; + private static final String SCHEMA_STR = "struct"; + + private static final int ROWS_PER_BATCH = 10; + private static final int NUM_BATCHES = 3; + private static final int TOTAL_ROWS = ROWS_PER_BATCH * NUM_BATCHES; // 30 + + private static final long FILTER_TIMESTAMP = 150; + // timestamps are 10,20,...,300 → rows 0-14 have ts <= 150 → 15 rows pass + private static final int EXPECTED_FILTERED_ROWS = 15; + + private static Storage storage; + + public static void main(String[] args) throws Exception + { + storage = StorageFactory.Instance().getStorage("file"); + setup(); + try + { + writeFileWithHiddenColumn(); + writeFileWithoutHiddenColumn(); + + testT1_EmptyProjection_ExposeOnly(); + testT2_EmptyProjection_FilterAndExpose(); + testT3_WithProjection_FilterAndExpose(); + testT4_WithProjection_ExposeOnly(); + testT5_MultiRG_StrideCorrectness(); + testT6_NoHiddenColumn_ValidTimestamp(); + testT7_EmptyProjection_FilterNoExpose(); + testT8_ExposeNoHiddenColumn_ThrowsIOException(); + testT9_FilterNoExpose_HiddenVectorNull(); + testT10_ApplyFilter_SyncsHiddenColumn(); + + System.out.println("\n=== All 10 tests passed! ==="); + } finally + { + cleanup(); + } + } + + // ======================= helpers ======================= + + private static void setup() throws IOException + { + Files.createDirectories(Paths.get(TEST_DIR)); + Files.deleteIfExists(Paths.get(FILE_WITH_HIDDEN)); + Files.deleteIfExists(Paths.get(FILE_NO_HIDDEN)); + } + + private static void cleanup() throws IOException + { + Files.deleteIfExists(Paths.get(FILE_WITH_HIDDEN)); + Files.deleteIfExists(Paths.get(FILE_NO_HIDDEN)); + } + + private static void check(boolean condition, String message) + { + if (!condition) + { + throw new AssertionError(message); + } + } + + private static long expectedTimestamp(int globalRowIdx) + { + return (globalRowIdx + 1) * 10L; + } + + // ======================= write test files ======================= + + private static void writeFileWithHiddenColumn() throws Exception + { + TypeDescription schema = TypeDescription.fromString(SCHEMA_STR); + PixelsWriter writer = PixelsWriterImpl.newBuilder() + .setSchema(schema) + .setHasHiddenColumn(true) + .setPixelStride(10) + .setRowGroupSize(1) // minimal → forces every batch into its own row group + .setStorage(storage) + .setPath(FILE_WITH_HIDDEN) + .setBlockSize(256 * 1024) + .setReplication((short) 1) + .setBlockPadding(false) + .setEncodingLevel(EncodingLevel.EL2) + .setCompressionBlockSize(1) + .setNullsPadding(false) + .build(); + + for (int batch = 0; batch < NUM_BATCHES; batch++) + { + VectorizedRowBatch rowBatch = schema.createRowBatchWithHiddenColumn(); + IntColumnVector x = (IntColumnVector) rowBatch.cols[0]; + IntColumnVector y = (IntColumnVector) rowBatch.cols[1]; + LongColumnVector hidden = (LongColumnVector) rowBatch.cols[2]; + + for (int i = 0; i < ROWS_PER_BATCH; i++) + { + int g = batch * ROWS_PER_BATCH + i; + int row = rowBatch.size++; + x.vector[row] = g * 100; + x.isNull[row] = false; + y.vector[row] = g * 200; + y.isNull[row] = false; + hidden.vector[row] = expectedTimestamp(g); + hidden.isNull[row] = false; + } + writer.addRowBatch(rowBatch); + } + writer.close(); + System.out.println("Written file WITH hidden column: " + FILE_WITH_HIDDEN); + } + + private static void writeFileWithoutHiddenColumn() throws Exception + { + TypeDescription schema = TypeDescription.fromString(SCHEMA_STR); + PixelsWriter writer = PixelsWriterImpl.newBuilder() + .setSchema(schema) + .setHasHiddenColumn(false) + .setPixelStride(10000) + .setRowGroupSize(64 * 1024 * 1024) + .setStorage(storage) + .setPath(FILE_NO_HIDDEN) + .setBlockSize(256 * 1024) + .setReplication((short) 1) + .setBlockPadding(false) + .setEncodingLevel(EncodingLevel.EL2) + .setCompressionBlockSize(1) + .setNullsPadding(false) + .build(); + + VectorizedRowBatch rowBatch = schema.createRowBatch(ROWS_PER_BATCH); + IntColumnVector x = (IntColumnVector) rowBatch.cols[0]; + IntColumnVector y = (IntColumnVector) rowBatch.cols[1]; + for (int i = 0; i < ROWS_PER_BATCH; i++) + { + int row = rowBatch.size++; + x.vector[row] = i * 100; + x.isNull[row] = false; + y.vector[row] = i * 200; + y.isNull[row] = false; + } + writer.addRowBatch(rowBatch); + writer.close(); + System.out.println("Written file WITHOUT hidden column: " + FILE_NO_HIDDEN); + } + + // ======================= tests ======================= + + /** + * T1: empty projection + expose only (no filter). + * Enters else-branch, reads hidden column directly into hiddenColumnVector. + * All 30 rows returned with correct timestamps. + */ + private static void testT1_EmptyProjection_ExposeOnly() throws Exception + { + System.out.println("\n--- T1: Empty projection + expose only (no filter) ---"); + PixelsReader reader = PixelsReaderImpl.newBuilder() + .setStorage(storage).setPath(FILE_WITH_HIDDEN) + .setPixelsFooterCache(new PixelsFooterCache()).build(); + + PixelsReaderOption option = new PixelsReaderOption(); + option.includeCols(new String[0]); + option.exposeHiddenColumn(true); + option.skipCorruptRecords(true); + option.tolerantSchemaEvolution(true); + + PixelsRecordReader rr = reader.read(option); + VectorizedRowBatch batch = rr.readBatch(TOTAL_ROWS + 100); + + check(batch.cols.length == 0, "T1: expected 0 user columns, got " + batch.cols.length); + check(batch.size == TOTAL_ROWS, "T1: expected " + TOTAL_ROWS + " rows, got " + batch.size); + LongColumnVector hv = batch.getHiddenColumnVector(); + check(hv != null, "T1: hiddenColumnVector should not be null"); + for (int i = 0; i < TOTAL_ROWS; i++) + { + check(hv.vector[i] == expectedTimestamp(i), + "T1: row " + i + " ts expected " + expectedTimestamp(i) + ", got " + hv.vector[i]); + } + rr.close(); + reader.close(); + System.out.println("T1 passed!"); + } + + /** + * T2: empty projection + filter + expose. + * Enters if-branch, filters by timestamp, copies selected timestamps. + */ + private static void testT2_EmptyProjection_FilterAndExpose() throws Exception + { + System.out.println("\n--- T2: Empty projection + filter + expose ---"); + PixelsReader reader = PixelsReaderImpl.newBuilder() + .setStorage(storage).setPath(FILE_WITH_HIDDEN) + .setPixelsFooterCache(new PixelsFooterCache()).build(); + + PixelsReaderOption option = new PixelsReaderOption(); + option.includeCols(new String[0]); + option.exposeHiddenColumn(true); + option.transId(0); + option.transTimestamp(FILTER_TIMESTAMP); + option.skipCorruptRecords(true); + option.tolerantSchemaEvolution(true); + + PixelsRecordReader rr = reader.read(option); + VectorizedRowBatch batch = rr.readBatch(TOTAL_ROWS + 100); + + check(batch.size == EXPECTED_FILTERED_ROWS, + "T2: expected " + EXPECTED_FILTERED_ROWS + " rows, got " + batch.size); + check(batch.cols.length == 0, "T2: expected 0 user columns, got " + batch.cols.length); + LongColumnVector hv = batch.getHiddenColumnVector(); + check(hv != null, "T2: hiddenColumnVector should not be null"); + for (int i = 0; i < batch.size; i++) + { + check(hv.vector[i] == expectedTimestamp(i), + "T2: row " + i + " ts expected " + expectedTimestamp(i) + ", got " + hv.vector[i]); + check(hv.vector[i] <= FILTER_TIMESTAMP, + "T2: row " + i + " ts " + hv.vector[i] + " should be <= " + FILTER_TIMESTAMP); + } + rr.close(); + reader.close(); + System.out.println("T2 passed!"); + } + + /** + * T3: with projection + filter + expose. + * Verifies user columns and hidden column are row-aligned after filtering. + */ + private static void testT3_WithProjection_FilterAndExpose() throws Exception + { + System.out.println("\n--- T3: With projection + filter + expose ---"); + PixelsReader reader = PixelsReaderImpl.newBuilder() + .setStorage(storage).setPath(FILE_WITH_HIDDEN) + .setPixelsFooterCache(new PixelsFooterCache()).build(); + + PixelsReaderOption option = new PixelsReaderOption(); + option.includeCols(new String[]{"x", "y"}); + option.exposeHiddenColumn(true); + option.transId(0); + option.transTimestamp(FILTER_TIMESTAMP); + option.skipCorruptRecords(true); + option.tolerantSchemaEvolution(true); + + PixelsRecordReader rr = reader.read(option); + VectorizedRowBatch batch = rr.readBatch(TOTAL_ROWS + 100); + + check(batch.size == EXPECTED_FILTERED_ROWS, + "T3: expected " + EXPECTED_FILTERED_ROWS + " rows, got " + batch.size); + check(batch.cols.length == 2, "T3: expected 2 user columns, got " + batch.cols.length); + + IntColumnVector xCol = (IntColumnVector) batch.cols[0]; + IntColumnVector yCol = (IntColumnVector) batch.cols[1]; + LongColumnVector hv = batch.getHiddenColumnVector(); + check(hv != null, "T3: hiddenColumnVector should not be null"); + + for (int i = 0; i < batch.size; i++) + { + check(xCol.vector[i] == i * 100L, + "T3: row " + i + " x expected " + (i * 100L) + ", got " + xCol.vector[i]); + check(yCol.vector[i] == i * 200L, + "T3: row " + i + " y expected " + (i * 200L) + ", got " + yCol.vector[i]); + check(hv.vector[i] == expectedTimestamp(i), + "T3: row " + i + " ts expected " + expectedTimestamp(i) + ", got " + hv.vector[i]); + } + rr.close(); + reader.close(); + System.out.println("T3 passed!"); + } + + /** + * T4: with projection + expose only (no filter). + * Enters else-branch, reads all rows with user columns and hidden column. + */ + private static void testT4_WithProjection_ExposeOnly() throws Exception + { + System.out.println("\n--- T4: With projection + expose only (no filter) ---"); + PixelsReader reader = PixelsReaderImpl.newBuilder() + .setStorage(storage).setPath(FILE_WITH_HIDDEN) + .setPixelsFooterCache(new PixelsFooterCache()).build(); + + PixelsReaderOption option = new PixelsReaderOption(); + option.includeCols(new String[]{"x"}); + option.exposeHiddenColumn(true); + option.skipCorruptRecords(true); + option.tolerantSchemaEvolution(true); + + PixelsRecordReader rr = reader.read(option); + VectorizedRowBatch batch = rr.readBatch(TOTAL_ROWS + 100); + + check(batch.size == TOTAL_ROWS, "T4: expected " + TOTAL_ROWS + " rows, got " + batch.size); + check(batch.cols.length == 1, "T4: expected 1 user column, got " + batch.cols.length); + + IntColumnVector xCol = (IntColumnVector) batch.cols[0]; + LongColumnVector hv = batch.getHiddenColumnVector(); + check(hv != null, "T4: hiddenColumnVector should not be null"); + + for (int i = 0; i < batch.size; i++) + { + check(xCol.vector[i] == i * 100L, + "T4: row " + i + " x expected " + (i * 100L) + ", got " + xCol.vector[i]); + check(hv.vector[i] == expectedTimestamp(i), + "T4: row " + i + " ts expected " + expectedTimestamp(i) + ", got " + hv.vector[i]); + } + rr.close(); + reader.close(); + System.out.println("T4 passed!"); + } + + /** + * T5: multi-RG stride correctness. + * Uses a small batch size (7) so that readBatch must cross RG boundaries. + * Reads with filter (if-branch) to exercise the fixed stride computation. + * Verifies every row's data is correct across all RGs. + */ + private static void testT5_MultiRG_StrideCorrectness() throws Exception + { + System.out.println("\n--- T5: Multi-RG stride correctness ---"); + PixelsReader reader = PixelsReaderImpl.newBuilder() + .setStorage(storage).setPath(FILE_WITH_HIDDEN) + .setPixelsFooterCache(new PixelsFooterCache()).build(); + + int rgNum = reader.getRowGroupNum(); + System.out.println(" Row groups in file: " + rgNum); + check(rgNum >= 2, "T5: expected >= 2 row groups, got " + rgNum); + + PixelsReaderOption option = new PixelsReaderOption(); + option.includeCols(new String[]{"x", "y"}); + option.exposeHiddenColumn(true); + option.transId(0); + option.transTimestamp(Long.MAX_VALUE); // accept all rows, but still enters if-branch + option.skipCorruptRecords(true); + option.tolerantSchemaEvolution(true); + + PixelsRecordReader rr = reader.read(option); + int totalRead = 0; + while (true) + { + VectorizedRowBatch batch = rr.readBatch(7); // odd size to cross RG boundaries + if (batch.size == 0 && batch.endOfFile) + { + break; + } + + IntColumnVector xCol = (IntColumnVector) batch.cols[0]; + LongColumnVector hv = batch.getHiddenColumnVector(); + check(hv != null, "T5: hiddenColumnVector should not be null in batch starting at row " + totalRead); + + for (int i = 0; i < batch.size; i++) + { + int g = totalRead + i; + check(xCol.vector[i] == g * 100L, + "T5: globalRow " + g + " x expected " + (g * 100L) + ", got " + xCol.vector[i]); + check(hv.vector[i] == expectedTimestamp(g), + "T5: globalRow " + g + " ts expected " + expectedTimestamp(g) + ", got " + hv.vector[i]); + } + totalRead += batch.size; + if (batch.endOfFile) + { + break; + } + } + check(totalRead == TOTAL_ROWS, "T5: expected " + TOTAL_ROWS + " total rows, got " + totalRead); + rr.close(); + reader.close(); + System.out.println("T5 passed! (read " + totalRead + " rows across multiple RGs)"); + } + + /** + * T6: no hidden column + valid transTimestamp. + * After the fix, this enters the else-branch (no filtering needed). + * Must not crash, data must be correct. + */ + private static void testT6_NoHiddenColumn_ValidTimestamp() throws Exception + { + System.out.println("\n--- T6: No hidden column + valid transTimestamp ---"); + PixelsReader reader = PixelsReaderImpl.newBuilder() + .setStorage(storage).setPath(FILE_NO_HIDDEN) + .setPixelsFooterCache(new PixelsFooterCache()).build(); + + PixelsReaderOption option = new PixelsReaderOption(); + option.includeCols(new String[]{"x", "y"}); + option.transId(0); + option.transTimestamp(100); + option.skipCorruptRecords(true); + option.tolerantSchemaEvolution(true); + + PixelsRecordReader rr = reader.read(option); + VectorizedRowBatch batch = rr.readBatch(100); + + check(batch.size == ROWS_PER_BATCH, + "T6: expected " + ROWS_PER_BATCH + " rows, got " + batch.size); + check(batch.getHiddenColumnVector() == null, + "T6: hiddenColumnVector should be null (exposeHiddenColumn not set)"); + + IntColumnVector xCol = (IntColumnVector) batch.cols[0]; + for (int i = 0; i < batch.size; i++) + { + check(xCol.vector[i] == i * 100L, + "T6: row " + i + " x expected " + (i * 100L) + ", got " + xCol.vector[i]); + } + rr.close(); + reader.close(); + System.out.println("T6 passed!"); + } + + /** + * T7: empty projection + filter + no expose. + * Equivalent to count(*) with timestamp filter. + * filterByHiddenTimestamp=true, needReadHiddenColumn=true, exposeHiddenColumn=false. + * Enters if-branch, filters rows, but hiddenColumnVector stays null. + */ + private static void testT7_EmptyProjection_FilterNoExpose() throws Exception + { + System.out.println("\n--- T7: Empty projection + filter + no expose (filtered count(*)) ---"); + PixelsReader reader = PixelsReaderImpl.newBuilder() + .setStorage(storage).setPath(FILE_WITH_HIDDEN) + .setPixelsFooterCache(new PixelsFooterCache()).build(); + + PixelsReaderOption option = new PixelsReaderOption(); + option.includeCols(new String[0]); + option.transId(0); + option.transTimestamp(FILTER_TIMESTAMP); + option.skipCorruptRecords(true); + option.tolerantSchemaEvolution(true); + + PixelsRecordReader rr = reader.read(option); + VectorizedRowBatch batch = rr.readBatch(TOTAL_ROWS + 100); + + check(batch.cols.length == 0, "T7: expected 0 user columns, got " + batch.cols.length); + check(batch.size == EXPECTED_FILTERED_ROWS, + "T7: expected " + EXPECTED_FILTERED_ROWS + " rows, got " + batch.size); + check(batch.getHiddenColumnVector() == null, + "T7: hiddenColumnVector should be null when exposeHiddenColumn is false"); + rr.close(); + reader.close(); + System.out.println("T7 passed!"); + } + + /** + * T8: exposeHiddenColumn=true on a file without hidden column. + * Constructor must throw IOException. + */ + private static void testT8_ExposeNoHiddenColumn_ThrowsIOException() throws Exception + { + System.out.println("\n--- T8: Expose + no hidden column file → IOException ---"); + PixelsReader reader = PixelsReaderImpl.newBuilder() + .setStorage(storage).setPath(FILE_NO_HIDDEN) + .setPixelsFooterCache(new PixelsFooterCache()).build(); + + PixelsReaderOption option = new PixelsReaderOption(); + option.includeCols(new String[]{"x"}); + option.exposeHiddenColumn(true); + option.skipCorruptRecords(true); + option.tolerantSchemaEvolution(true); + + try + { + reader.read(option); + check(false, "T8: should have thrown IOException"); + } catch (IOException e) + { + check(e.getMessage().contains("no hidden column"), + "T8: exception should mention 'no hidden column', got: " + e.getMessage()); + System.out.println(" Caught expected IOException: " + e.getMessage()); + } + reader.close(); + System.out.println("T8 passed!"); + } + + /** + * T9: filter + no expose (original behavior regression). + * hiddenColumnVector must be null in the result batch. + */ + private static void testT9_FilterNoExpose_HiddenVectorNull() throws Exception + { + System.out.println("\n--- T9: Filter + no expose → hiddenColumnVector null ---"); + PixelsReader reader = PixelsReaderImpl.newBuilder() + .setStorage(storage).setPath(FILE_WITH_HIDDEN) + .setPixelsFooterCache(new PixelsFooterCache()).build(); + + PixelsReaderOption option = new PixelsReaderOption(); + option.includeCols(new String[]{"x", "y"}); + option.transId(0); + option.transTimestamp(FILTER_TIMESTAMP); + option.skipCorruptRecords(true); + option.tolerantSchemaEvolution(true); + // exposeHiddenColumn defaults to false + + PixelsRecordReader rr = reader.read(option); + VectorizedRowBatch batch = rr.readBatch(TOTAL_ROWS + 100); + + check(batch.size == EXPECTED_FILTERED_ROWS, + "T9: expected " + EXPECTED_FILTERED_ROWS + " rows, got " + batch.size); + check(batch.getHiddenColumnVector() == null, + "T9: hiddenColumnVector should be null when exposeHiddenColumn is false"); + + IntColumnVector xCol = (IntColumnVector) batch.cols[0]; + for (int i = 0; i < batch.size; i++) + { + check(xCol.vector[i] == i * 100L, + "T9: row " + i + " x expected " + (i * 100L) + ", got " + xCol.vector[i]); + } + rr.close(); + reader.close(); + System.out.println("T9 passed!"); + } + + /** + * T10: VectorizedRowBatch.applyFilter must sync hiddenColumnVector. + * Read all rows with expose, then apply an external filter keeping even-indexed rows. + */ + private static void testT10_ApplyFilter_SyncsHiddenColumn() throws Exception + { + System.out.println("\n--- T10: applyFilter syncs hiddenColumnVector ---"); + PixelsReader reader = PixelsReaderImpl.newBuilder() + .setStorage(storage).setPath(FILE_WITH_HIDDEN) + .setPixelsFooterCache(new PixelsFooterCache()).build(); + + PixelsReaderOption option = new PixelsReaderOption(); + option.includeCols(new String[]{"x"}); + option.exposeHiddenColumn(true); + option.skipCorruptRecords(true); + option.tolerantSchemaEvolution(true); + + PixelsRecordReader rr = reader.read(option); + VectorizedRowBatch batch = rr.readBatch(TOTAL_ROWS + 100); + check(batch.size == TOTAL_ROWS, "T10: expected " + TOTAL_ROWS + " rows before filter"); + + Bitmap filter = new Bitmap(TOTAL_ROWS, false); + for (int i = 0; i < TOTAL_ROWS; i += 2) + { + filter.set(i); // keep even-indexed rows: 0, 2, 4, ... + } + batch.applyFilter(filter); + + int expectedAfterFilter = (TOTAL_ROWS + 1) / 2; // ceil(30/2) = 15 + check(batch.size == expectedAfterFilter, + "T10: expected " + expectedAfterFilter + " rows after filter, got " + batch.size); + + IntColumnVector xCol = (IntColumnVector) batch.cols[0]; + LongColumnVector hv = batch.getHiddenColumnVector(); + check(hv != null, "T10: hiddenColumnVector should survive applyFilter"); + + for (int i = 0; i < batch.size; i++) + { + int originalIdx = i * 2; + check(xCol.vector[i] == originalIdx * 100L, + "T10: filtered row " + i + " x expected " + (originalIdx * 100L) + ", got " + xCol.vector[i]); + check(hv.vector[i] == expectedTimestamp(originalIdx), + "T10: filtered row " + i + " ts expected " + expectedTimestamp(originalIdx) + ", got " + hv.vector[i]); + } + rr.close(); + reader.close(); + System.out.println("T10 passed!"); + } +} diff --git a/pixels-example/src/main/java/io/pixelsdb/pixels/example/core/TestPixelsReadWrite.java b/pixels-example/src/main/java/io/pixelsdb/pixels/example/core/TestPixelsReadWrite.java new file mode 100644 index 0000000000..18b758a23c --- /dev/null +++ b/pixels-example/src/main/java/io/pixelsdb/pixels/example/core/TestPixelsReadWrite.java @@ -0,0 +1,273 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the Affero GNU General Public + * License along with Pixels. If not, see + * . + */ +package io.pixelsdb.pixels.example.core; + +import io.pixelsdb.pixels.common.physical.Storage; +import io.pixelsdb.pixels.common.physical.StorageFactory; +import io.pixelsdb.pixels.core.PixelsFooterCache; +import io.pixelsdb.pixels.core.PixelsReader; +import io.pixelsdb.pixels.core.PixelsReaderImpl; +import io.pixelsdb.pixels.core.PixelsWriter; +import io.pixelsdb.pixels.core.PixelsWriterImpl; +import io.pixelsdb.pixels.core.TypeDescription; +import io.pixelsdb.pixels.core.encoding.EncodingLevel; +import io.pixelsdb.pixels.core.reader.PixelsReaderOption; +import io.pixelsdb.pixels.core.reader.PixelsRecordReader; +import io.pixelsdb.pixels.core.vector.BinaryColumnVector; +import io.pixelsdb.pixels.core.vector.ByteColumnVector; +import io.pixelsdb.pixels.core.vector.DateColumnVector; +import io.pixelsdb.pixels.core.vector.DecimalColumnVector; +import io.pixelsdb.pixels.core.vector.DoubleColumnVector; +import io.pixelsdb.pixels.core.vector.FloatColumnVector; +import io.pixelsdb.pixels.core.vector.IntColumnVector; +import io.pixelsdb.pixels.core.vector.LongColumnVector; +import io.pixelsdb.pixels.core.vector.ShortColumnVector; +import io.pixelsdb.pixels.core.vector.TimeColumnVector; +import io.pixelsdb.pixels.core.vector.TimestampColumnVector; +import io.pixelsdb.pixels.core.vector.VectorizedRowBatch; + +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * End-to-end example and self-check that writes every supported column type to a local + * Pixels file and reads it back, verifying each value (including nulls) round-trips. + *

+ * This single class exercises the writer, the reader, and all column vectors together. + * The file is written across multiple row groups and read back with a batch size that + * crosses row-group boundaries, so multi-row-group stride handling is covered as well. + *

+ * For minimal, single-purpose read/write snippets, see {@code TestPixelsReader} and + * {@code TestPixelsWriter}. + */ +public class TestPixelsReadWrite +{ + private static final String SCHEMA_STRING = + "struct"; + + private static final int ROWS_PER_GROUP = 10; + private static final int GROUP_COUNT = 3; + private static final int TOTAL_ROWS = ROWS_PER_GROUP * GROUP_COUNT; + private static final int READ_BATCH_SIZE = 7; // odd size to cross row-group boundaries + + public static void main(String[] args) throws Exception + { + Storage storage = StorageFactory.Instance().getStorage(Storage.Scheme.file); + Path path = Files.createTempFile("pixels-read-write-", ".pxl"); + Files.deleteIfExists(path); + try + { + writeFile(storage, path.toString()); + readAndVerify(storage, path.toString()); + System.out.println("TestPixelsReadWrite passed: " + TOTAL_ROWS + " rows round-tripped across " + + GROUP_COUNT + " row groups."); + } + finally + { + Files.deleteIfExists(path); + } + } + + // ---- deterministic expected values (single source of truth for write and verify) ---- + + private static boolean isNullRow(int g) + { + return g % 6 == 5; // rows 5, 11, 17, 23, 29 are null + } + + private static byte expectedBool(int g) { return (byte) (g % 2); } + private static short expectedShort(int g) { return (short) (g - 15); } + private static int expectedInt(int g) { return g * 1000; } + private static long expectedLong(int g) { return (long) g * 1_000_000L; } + private static float expectedFloat(int g) { return g * 1.5f; } + private static double expectedDouble(int g) { return g * 2.25d; } + private static long expectedDecimalUnscaled(int g) { return g * 1000L + 123L; } // decimal(12,3) + private static int expectedDate(int g) { return 19000 + g; } // days since epoch + private static int expectedTimeMillis(int g) { return g * 1000; } // millis in day, precision 3 + private static long expectedTsMicros(int g) { return (long) g * 1_000_000L; } // whole seconds, precision 3 safe + private static String expectedStr(int g) { return "s" + g; } + + // ------------------------------- write ------------------------------- + + private static void writeFile(Storage storage, String filePath) throws Exception + { + TypeDescription schema = TypeDescription.fromString(SCHEMA_STRING); + PixelsWriter writer = PixelsWriterImpl.newBuilder() + .setSchema(schema) + .setPixelStride(ROWS_PER_GROUP) + .setRowGroupSize(1) // minimal → forces each added batch into its own row group + .setStorage(storage) + .setPath(filePath) + .setBlockSize(256 * 1024) + .setReplication((short) 1) + .setBlockPadding(false) + .setEncodingLevel(EncodingLevel.EL2) + .setCompressionBlockSize(1) + .setNullsPadding(true) + .build(); + + for (int group = 0; group < GROUP_COUNT; group++) + { + VectorizedRowBatch rowBatch = schema.createRowBatch(ROWS_PER_GROUP); + ByteColumnVector cBool = (ByteColumnVector) rowBatch.cols[0]; + ShortColumnVector cShort = (ShortColumnVector) rowBatch.cols[1]; + IntColumnVector cInt = (IntColumnVector) rowBatch.cols[2]; + LongColumnVector cLong = (LongColumnVector) rowBatch.cols[3]; + FloatColumnVector cFloat = (FloatColumnVector) rowBatch.cols[4]; + DoubleColumnVector cDouble = (DoubleColumnVector) rowBatch.cols[5]; + DecimalColumnVector cDec = (DecimalColumnVector) rowBatch.cols[6]; + DateColumnVector cDate = (DateColumnVector) rowBatch.cols[7]; + TimeColumnVector cTime = (TimeColumnVector) rowBatch.cols[8]; + TimestampColumnVector cTs = (TimestampColumnVector) rowBatch.cols[9]; + BinaryColumnVector cStr = (BinaryColumnVector) rowBatch.cols[10]; + + for (int i = 0; i < ROWS_PER_GROUP; i++) + { + int g = group * ROWS_PER_GROUP + i; + int row = rowBatch.size++; + if (isNullRow(g)) + { + cBool.isNull[row] = true; cBool.noNulls = false; + cShort.isNull[row] = true; cShort.noNulls = false; + cInt.isNull[row] = true; cInt.noNulls = false; + cLong.isNull[row] = true; cLong.noNulls = false; + cFloat.isNull[row] = true; cFloat.noNulls = false; + cDouble.isNull[row] = true; cDouble.noNulls = false; + cDec.isNull[row] = true; cDec.noNulls = false; + cDate.isNull[row] = true; cDate.noNulls = false; + cTime.isNull[row] = true; cTime.noNulls = false; + cTs.isNull[row] = true; cTs.noNulls = false; + cStr.isNull[row] = true; cStr.noNulls = false; + continue; + } + cBool.vector[row] = expectedBool(g); cBool.isNull[row] = false; + cShort.vector[row] = expectedShort(g); cShort.isNull[row] = false; + cInt.vector[row] = expectedInt(g); cInt.isNull[row] = false; + cLong.vector[row] = expectedLong(g); cLong.isNull[row] = false; + cFloat.vector[row] = Float.floatToIntBits(expectedFloat(g)); cFloat.isNull[row] = false; + cDouble.vector[row] = Double.doubleToLongBits(expectedDouble(g)); cDouble.isNull[row] = false; + cDec.vector[row] = expectedDecimalUnscaled(g); cDec.isNull[row] = false; + cDate.set(row, expectedDate(g)); + cTime.set(row, expectedTimeMillis(g)); + cTs.set(row, expectedTsMicros(g)); + cStr.setVal(row, expectedStr(g).getBytes()); cStr.isNull[row] = false; + } + writer.addRowBatch(rowBatch); + rowBatch.reset(); + } + writer.close(); + } + + // ------------------------------- read + verify ------------------------------- + + private static void readAndVerify(Storage storage, String filePath) throws Exception + { + try (PixelsReader reader = PixelsReaderImpl.newBuilder() + .setStorage(storage) + .setPath(filePath) + .setPixelsFooterCache(new PixelsFooterCache()) + .build()) + { + check(reader.getRowGroupNum() == GROUP_COUNT, + "expected " + GROUP_COUNT + " row groups, got " + reader.getRowGroupNum()); + + PixelsReaderOption option = new PixelsReaderOption(); + option.skipCorruptRecords(true); + option.tolerantSchemaEvolution(true); + option.includeCols(reader.getFileSchema().getFieldNames().toArray(new String[0])); + + PixelsRecordReader recordReader = reader.read(option); + int totalRead = 0; + while (true) + { + VectorizedRowBatch batch = recordReader.readBatch(READ_BATCH_SIZE); + if (batch.size == 0 && batch.endOfFile) + { + break; + } + verifyBatch(batch, totalRead); + totalRead += batch.size; + if (batch.endOfFile) + { + break; + } + } + recordReader.close(); + check(totalRead == TOTAL_ROWS, "expected " + TOTAL_ROWS + " rows, got " + totalRead); + } + } + + private static void verifyBatch(VectorizedRowBatch batch, int rowOffset) + { + ByteColumnVector cBool = (ByteColumnVector) batch.cols[0]; + ShortColumnVector cShort = (ShortColumnVector) batch.cols[1]; + IntColumnVector cInt = (IntColumnVector) batch.cols[2]; + LongColumnVector cLong = (LongColumnVector) batch.cols[3]; + FloatColumnVector cFloat = (FloatColumnVector) batch.cols[4]; + DoubleColumnVector cDouble = (DoubleColumnVector) batch.cols[5]; + DecimalColumnVector cDec = (DecimalColumnVector) batch.cols[6]; + DateColumnVector cDate = (DateColumnVector) batch.cols[7]; + TimeColumnVector cTime = (TimeColumnVector) batch.cols[8]; + TimestampColumnVector cTs = (TimestampColumnVector) batch.cols[9]; + BinaryColumnVector cStr = (BinaryColumnVector) batch.cols[10]; + + for (int i = 0; i < batch.size; i++) + { + int g = rowOffset + i; + if (isNullRow(g)) + { + check(cBool.isNull[i], "row " + g + " c_bool should be null"); + check(cShort.isNull[i], "row " + g + " c_short should be null"); + check(cInt.isNull[i], "row " + g + " c_int should be null"); + check(cLong.isNull[i], "row " + g + " c_long should be null"); + check(cFloat.isNull[i], "row " + g + " c_float should be null"); + check(cDouble.isNull[i], "row " + g + " c_double should be null"); + check(cDec.isNull[i], "row " + g + " c_dec should be null"); + check(cDate.isNull[i], "row " + g + " c_date should be null"); + check(cTime.isNull[i], "row " + g + " c_time should be null"); + check(cTs.isNull[i], "row " + g + " c_ts should be null"); + check(cStr.isNull[i], "row " + g + " c_str should be null"); + continue; + } + check(cBool.vector[i] == expectedBool(g), "row " + g + " c_bool"); + check(cShort.vector[i] == expectedShort(g), "row " + g + " c_short"); + check(cInt.vector[i] == expectedInt(g), "row " + g + " c_int"); + check(cLong.vector[i] == expectedLong(g), "row " + g + " c_long"); + check(Float.intBitsToFloat(cFloat.vector[i]) == expectedFloat(g), "row " + g + " c_float"); + check(Double.longBitsToDouble(cDouble.vector[i]) == expectedDouble(g), "row " + g + " c_double"); + check(cDec.vector[i] == expectedDecimalUnscaled(g), "row " + g + " c_dec"); + check(cDate.dates[i] == expectedDate(g), "row " + g + " c_date"); + check(cTime.times[i] == expectedTimeMillis(g), "row " + g + " c_time"); + check(cTs.times[i] == expectedTsMicros(g), "row " + g + " c_ts"); + check(expectedStr(g).equals(new String(cStr.vector[i], cStr.start[i], cStr.lens[i])), + "row " + g + " c_str"); + } + } + + private static void check(boolean condition, String message) + { + if (!condition) + { + throw new AssertionError(message); + } + } +} diff --git a/pixels-example/src/main/java/io/pixelsdb/pixels/example/core/TestPixelsReader.java b/pixels-example/src/main/java/io/pixelsdb/pixels/example/core/TestPixelsReader.java index 0f268e06aa..6928e6b850 100644 --- a/pixels-example/src/main/java/io/pixelsdb/pixels/example/core/TestPixelsReader.java +++ b/pixels-example/src/main/java/io/pixelsdb/pixels/example/core/TestPixelsReader.java @@ -21,588 +21,60 @@ import io.pixelsdb.pixels.common.physical.Storage; import io.pixelsdb.pixels.common.physical.StorageFactory; -import io.pixelsdb.pixels.core.*; -import io.pixelsdb.pixels.core.encoding.EncodingLevel; +import io.pixelsdb.pixels.core.PixelsFooterCache; +import io.pixelsdb.pixels.core.PixelsReader; +import io.pixelsdb.pixels.core.PixelsReaderImpl; import io.pixelsdb.pixels.core.reader.PixelsReaderOption; import io.pixelsdb.pixels.core.reader.PixelsRecordReader; -import io.pixelsdb.pixels.core.utils.Bitmap; -import io.pixelsdb.pixels.core.vector.LongColumnVector; import io.pixelsdb.pixels.core.vector.VectorizedRowBatch; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; - /** - * Tests for hidden-timestamp-column expose/filter logic in PixelsRecordReaderImpl. + * Minimal example showing how to read a Pixels file from the local file system. + *

+ * Run {@code TestPixelsWriter} first to produce the file, then this example opens a + * {@link PixelsReader}, selects the columns to project via {@link PixelsReaderOption}, + * and iterates {@link VectorizedRowBatch}es until end of file. + *

+ * For an end-to-end write-then-read example with per-value verification across all + * column types, see {@code TestPixelsReadWrite}. * - * Covers: - * T1 empty projection + expose only (no filter) - * T2 empty projection + filter + expose - * T3 with projection + filter + expose - * T4 with projection + expose only (no filter) - * T5 multi-RG stride correctness (small batch across RG boundaries) - * T6 no hidden column + valid transTimestamp (must not crash) - * T7 empty projection + filter + no expose (count(*) with timestamp filter) - * T8 expose + no-hidden-column file (must throw IOException) - * T9 filter + no expose (hiddenColumnVector must be null) - * T10 VectorizedRowBatch.applyFilter syncs hiddenColumnVector + * @author hank + * @create 2018-11-19 */ public class TestPixelsReader { - private static final String TEST_DIR = "/tmp/pixels_test_reader/"; - private static final String FILE_WITH_HIDDEN = TEST_DIR + "with_hidden.pxl"; - private static final String FILE_NO_HIDDEN = TEST_DIR + "no_hidden.pxl"; - private static final String SCHEMA_STR = "struct"; - - private static final int ROWS_PER_BATCH = 10; - private static final int NUM_BATCHES = 3; - private static final int TOTAL_ROWS = ROWS_PER_BATCH * NUM_BATCHES; // 30 - - private static final long FILTER_TIMESTAMP = 150; - // timestamps are 10,20,...,300 → rows 0-14 have ts <= 150 → 15 rows pass - private static final int EXPECTED_FILTERED_ROWS = 15; - - private static Storage storage; + private static final int BATCH_SIZE = 10000; public static void main(String[] args) throws Exception { - storage = StorageFactory.Instance().getStorage("file"); - setup(); - try - { - writeFileWithHiddenColumn(); - writeFileWithoutHiddenColumn(); - - testT1_EmptyProjection_ExposeOnly(); - testT2_EmptyProjection_FilterAndExpose(); - testT3_WithProjection_FilterAndExpose(); - testT4_WithProjection_ExposeOnly(); - testT5_MultiRG_StrideCorrectness(); - testT6_NoHiddenColumn_ValidTimestamp(); - testT7_EmptyProjection_FilterNoExpose(); - testT8_ExposeNoHiddenColumn_ThrowsIOException(); - testT9_FilterNoExpose_HiddenVectorNull(); - testT10_ApplyFilter_SyncsHiddenColumn(); - - System.out.println("\n=== All 10 tests passed! ==="); - } finally - { - cleanup(); - } - } - - // ======================= helpers ======================= - - private static void setup() throws IOException - { - Files.createDirectories(Paths.get(TEST_DIR)); - Files.deleteIfExists(Paths.get(FILE_WITH_HIDDEN)); - Files.deleteIfExists(Paths.get(FILE_NO_HIDDEN)); - } - - private static void cleanup() throws IOException - { - Files.deleteIfExists(Paths.get(FILE_WITH_HIDDEN)); - Files.deleteIfExists(Paths.get(FILE_NO_HIDDEN)); - } - - private static void check(boolean condition, String message) - { - if (!condition) - { - throw new AssertionError(message); - } - } - - private static long expectedTimestamp(int globalRowIdx) - { - return (globalRowIdx + 1) * 10L; - } - - // ======================= write test files ======================= + String pixelsFile = "/tmp/pixels-writer-example.pxl"; + Storage storage = StorageFactory.Instance().getStorage(Storage.Scheme.file); - private static void writeFileWithHiddenColumn() throws Exception - { - TypeDescription schema = TypeDescription.fromString(SCHEMA_STR); - PixelsWriter writer = PixelsWriterImpl.newBuilder() - .setSchema(schema) - .setHasHiddenColumn(true) - .setPixelStride(10) - .setRowGroupSize(1) // minimal → forces every batch into its own row group + try (PixelsReader reader = PixelsReaderImpl.newBuilder() .setStorage(storage) - .setPath(FILE_WITH_HIDDEN) - .setBlockSize(256 * 1024) - .setReplication((short) 1) - .setBlockPadding(false) - .setEncodingLevel(EncodingLevel.EL2) - .setCompressionBlockSize(1) - .setNullsPadding(false) - .build(); - - for (int batch = 0; batch < NUM_BATCHES; batch++) - { - VectorizedRowBatch rowBatch = schema.createRowBatchWithHiddenColumn(); - LongColumnVector x = (LongColumnVector) rowBatch.cols[0]; - LongColumnVector y = (LongColumnVector) rowBatch.cols[1]; - LongColumnVector hidden = (LongColumnVector) rowBatch.cols[2]; - - for (int i = 0; i < ROWS_PER_BATCH; i++) + .setPath(pixelsFile) + .setPixelsFooterCache(new PixelsFooterCache()) + .build()) + { + PixelsReaderOption option = new PixelsReaderOption(); + option.skipCorruptRecords(true); + option.tolerantSchemaEvolution(true); + // project all columns of the file schema + option.includeCols(reader.getFileSchema().getFieldNames().toArray(new String[0])); + + PixelsRecordReader recordReader = reader.read(option); + long totalRows = 0; + while (true) { - int g = batch * ROWS_PER_BATCH + i; - int row = rowBatch.size++; - x.vector[row] = g * 100L; - x.isNull[row] = false; - y.vector[row] = g * 200L; - y.isNull[row] = false; - hidden.vector[row] = expectedTimestamp(g); - hidden.isNull[row] = false; + VectorizedRowBatch rowBatch = recordReader.readBatch(BATCH_SIZE); + totalRows += rowBatch.size; + if (rowBatch.endOfFile) + { + break; + } } - writer.addRowBatch(rowBatch); - } - writer.close(); - System.out.println("Written file WITH hidden column: " + FILE_WITH_HIDDEN); - } - - private static void writeFileWithoutHiddenColumn() throws Exception - { - TypeDescription schema = TypeDescription.fromString(SCHEMA_STR); - PixelsWriter writer = PixelsWriterImpl.newBuilder() - .setSchema(schema) - .setHasHiddenColumn(false) - .setPixelStride(10000) - .setRowGroupSize(64 * 1024 * 1024) - .setStorage(storage) - .setPath(FILE_NO_HIDDEN) - .setBlockSize(256 * 1024) - .setReplication((short) 1) - .setBlockPadding(false) - .setEncodingLevel(EncodingLevel.EL2) - .setCompressionBlockSize(1) - .setNullsPadding(false) - .build(); - - VectorizedRowBatch rowBatch = schema.createRowBatch(ROWS_PER_BATCH); - LongColumnVector x = (LongColumnVector) rowBatch.cols[0]; - LongColumnVector y = (LongColumnVector) rowBatch.cols[1]; - for (int i = 0; i < ROWS_PER_BATCH; i++) - { - int row = rowBatch.size++; - x.vector[row] = i * 100L; - x.isNull[row] = false; - y.vector[row] = i * 200L; - y.isNull[row] = false; - } - writer.addRowBatch(rowBatch); - writer.close(); - System.out.println("Written file WITHOUT hidden column: " + FILE_NO_HIDDEN); - } - - // ======================= tests ======================= - - /** - * T1: empty projection + expose only (no filter). - * Enters else-branch, reads hidden column directly into hiddenColumnVector. - * All 30 rows returned with correct timestamps. - */ - private static void testT1_EmptyProjection_ExposeOnly() throws Exception - { - System.out.println("\n--- T1: Empty projection + expose only (no filter) ---"); - PixelsReader reader = PixelsReaderImpl.newBuilder() - .setStorage(storage).setPath(FILE_WITH_HIDDEN) - .setPixelsFooterCache(new PixelsFooterCache()).build(); - - PixelsReaderOption option = new PixelsReaderOption(); - option.includeCols(new String[0]); - option.exposeHiddenColumn(true); - option.skipCorruptRecords(true); - option.tolerantSchemaEvolution(true); - - PixelsRecordReader rr = reader.read(option); - VectorizedRowBatch batch = rr.readBatch(TOTAL_ROWS + 100); - - check(batch.cols.length == 0, "T1: expected 0 user columns, got " + batch.cols.length); - check(batch.size == TOTAL_ROWS, "T1: expected " + TOTAL_ROWS + " rows, got " + batch.size); - LongColumnVector hv = batch.getHiddenColumnVector(); - check(hv != null, "T1: hiddenColumnVector should not be null"); - for (int i = 0; i < TOTAL_ROWS; i++) - { - check(hv.vector[i] == expectedTimestamp(i), - "T1: row " + i + " ts expected " + expectedTimestamp(i) + ", got " + hv.vector[i]); - } - rr.close(); - reader.close(); - System.out.println("T1 passed!"); - } - - /** - * T2: empty projection + filter + expose. - * Enters if-branch, filters by timestamp, copies selected timestamps. - */ - private static void testT2_EmptyProjection_FilterAndExpose() throws Exception - { - System.out.println("\n--- T2: Empty projection + filter + expose ---"); - PixelsReader reader = PixelsReaderImpl.newBuilder() - .setStorage(storage).setPath(FILE_WITH_HIDDEN) - .setPixelsFooterCache(new PixelsFooterCache()).build(); - - PixelsReaderOption option = new PixelsReaderOption(); - option.includeCols(new String[0]); - option.exposeHiddenColumn(true); - option.transId(0); - option.transTimestamp(FILTER_TIMESTAMP); - option.skipCorruptRecords(true); - option.tolerantSchemaEvolution(true); - - PixelsRecordReader rr = reader.read(option); - VectorizedRowBatch batch = rr.readBatch(TOTAL_ROWS + 100); - - check(batch.size == EXPECTED_FILTERED_ROWS, - "T2: expected " + EXPECTED_FILTERED_ROWS + " rows, got " + batch.size); - check(batch.cols.length == 0, "T2: expected 0 user columns, got " + batch.cols.length); - LongColumnVector hv = batch.getHiddenColumnVector(); - check(hv != null, "T2: hiddenColumnVector should not be null"); - for (int i = 0; i < batch.size; i++) - { - check(hv.vector[i] == expectedTimestamp(i), - "T2: row " + i + " ts expected " + expectedTimestamp(i) + ", got " + hv.vector[i]); - check(hv.vector[i] <= FILTER_TIMESTAMP, - "T2: row " + i + " ts " + hv.vector[i] + " should be <= " + FILTER_TIMESTAMP); - } - rr.close(); - reader.close(); - System.out.println("T2 passed!"); - } - - /** - * T3: with projection + filter + expose. - * Verifies user columns and hidden column are row-aligned after filtering. - */ - private static void testT3_WithProjection_FilterAndExpose() throws Exception - { - System.out.println("\n--- T3: With projection + filter + expose ---"); - PixelsReader reader = PixelsReaderImpl.newBuilder() - .setStorage(storage).setPath(FILE_WITH_HIDDEN) - .setPixelsFooterCache(new PixelsFooterCache()).build(); - - PixelsReaderOption option = new PixelsReaderOption(); - option.includeCols(new String[]{"x", "y"}); - option.exposeHiddenColumn(true); - option.transId(0); - option.transTimestamp(FILTER_TIMESTAMP); - option.skipCorruptRecords(true); - option.tolerantSchemaEvolution(true); - - PixelsRecordReader rr = reader.read(option); - VectorizedRowBatch batch = rr.readBatch(TOTAL_ROWS + 100); - - check(batch.size == EXPECTED_FILTERED_ROWS, - "T3: expected " + EXPECTED_FILTERED_ROWS + " rows, got " + batch.size); - check(batch.cols.length == 2, "T3: expected 2 user columns, got " + batch.cols.length); - - LongColumnVector xCol = (LongColumnVector) batch.cols[0]; - LongColumnVector yCol = (LongColumnVector) batch.cols[1]; - LongColumnVector hv = batch.getHiddenColumnVector(); - check(hv != null, "T3: hiddenColumnVector should not be null"); - - for (int i = 0; i < batch.size; i++) - { - check(xCol.vector[i] == i * 100L, - "T3: row " + i + " x expected " + (i * 100L) + ", got " + xCol.vector[i]); - check(yCol.vector[i] == i * 200L, - "T3: row " + i + " y expected " + (i * 200L) + ", got " + yCol.vector[i]); - check(hv.vector[i] == expectedTimestamp(i), - "T3: row " + i + " ts expected " + expectedTimestamp(i) + ", got " + hv.vector[i]); - } - rr.close(); - reader.close(); - System.out.println("T3 passed!"); - } - - /** - * T4: with projection + expose only (no filter). - * Enters else-branch, reads all rows with user columns and hidden column. - */ - private static void testT4_WithProjection_ExposeOnly() throws Exception - { - System.out.println("\n--- T4: With projection + expose only (no filter) ---"); - PixelsReader reader = PixelsReaderImpl.newBuilder() - .setStorage(storage).setPath(FILE_WITH_HIDDEN) - .setPixelsFooterCache(new PixelsFooterCache()).build(); - - PixelsReaderOption option = new PixelsReaderOption(); - option.includeCols(new String[]{"x"}); - option.exposeHiddenColumn(true); - option.skipCorruptRecords(true); - option.tolerantSchemaEvolution(true); - - PixelsRecordReader rr = reader.read(option); - VectorizedRowBatch batch = rr.readBatch(TOTAL_ROWS + 100); - - check(batch.size == TOTAL_ROWS, "T4: expected " + TOTAL_ROWS + " rows, got " + batch.size); - check(batch.cols.length == 1, "T4: expected 1 user column, got " + batch.cols.length); - - LongColumnVector xCol = (LongColumnVector) batch.cols[0]; - LongColumnVector hv = batch.getHiddenColumnVector(); - check(hv != null, "T4: hiddenColumnVector should not be null"); - - for (int i = 0; i < batch.size; i++) - { - check(xCol.vector[i] == i * 100L, - "T4: row " + i + " x expected " + (i * 100L) + ", got " + xCol.vector[i]); - check(hv.vector[i] == expectedTimestamp(i), - "T4: row " + i + " ts expected " + expectedTimestamp(i) + ", got " + hv.vector[i]); - } - rr.close(); - reader.close(); - System.out.println("T4 passed!"); - } - - /** - * T5: multi-RG stride correctness. - * Uses a small batch size (7) so that readBatch must cross RG boundaries. - * Reads with filter (if-branch) to exercise the fixed stride computation. - * Verifies every row's data is correct across all RGs. - */ - private static void testT5_MultiRG_StrideCorrectness() throws Exception - { - System.out.println("\n--- T5: Multi-RG stride correctness ---"); - PixelsReader reader = PixelsReaderImpl.newBuilder() - .setStorage(storage).setPath(FILE_WITH_HIDDEN) - .setPixelsFooterCache(new PixelsFooterCache()).build(); - - int rgNum = reader.getRowGroupNum(); - System.out.println(" Row groups in file: " + rgNum); - check(rgNum >= 2, "T5: expected >= 2 row groups, got " + rgNum); - - PixelsReaderOption option = new PixelsReaderOption(); - option.includeCols(new String[]{"x", "y"}); - option.exposeHiddenColumn(true); - option.transId(0); - option.transTimestamp(Long.MAX_VALUE); // accept all rows, but still enters if-branch - option.skipCorruptRecords(true); - option.tolerantSchemaEvolution(true); - - PixelsRecordReader rr = reader.read(option); - int totalRead = 0; - while (true) - { - VectorizedRowBatch batch = rr.readBatch(7); // odd size to cross RG boundaries - if (batch.size == 0 && batch.endOfFile) - { - break; - } - - LongColumnVector xCol = (LongColumnVector) batch.cols[0]; - LongColumnVector hv = batch.getHiddenColumnVector(); - check(hv != null, "T5: hiddenColumnVector should not be null in batch starting at row " + totalRead); - - for (int i = 0; i < batch.size; i++) - { - int g = totalRead + i; - check(xCol.vector[i] == g * 100L, - "T5: globalRow " + g + " x expected " + (g * 100L) + ", got " + xCol.vector[i]); - check(hv.vector[i] == expectedTimestamp(g), - "T5: globalRow " + g + " ts expected " + expectedTimestamp(g) + ", got " + hv.vector[i]); - } - totalRead += batch.size; - if (batch.endOfFile) - { - break; - } - } - check(totalRead == TOTAL_ROWS, "T5: expected " + TOTAL_ROWS + " total rows, got " + totalRead); - rr.close(); - reader.close(); - System.out.println("T5 passed! (read " + totalRead + " rows across multiple RGs)"); - } - - /** - * T6: no hidden column + valid transTimestamp. - * After the fix, this enters the else-branch (no filtering needed). - * Must not crash, data must be correct. - */ - private static void testT6_NoHiddenColumn_ValidTimestamp() throws Exception - { - System.out.println("\n--- T6: No hidden column + valid transTimestamp ---"); - PixelsReader reader = PixelsReaderImpl.newBuilder() - .setStorage(storage).setPath(FILE_NO_HIDDEN) - .setPixelsFooterCache(new PixelsFooterCache()).build(); - - PixelsReaderOption option = new PixelsReaderOption(); - option.includeCols(new String[]{"x", "y"}); - option.transId(0); - option.transTimestamp(100); - option.skipCorruptRecords(true); - option.tolerantSchemaEvolution(true); - - PixelsRecordReader rr = reader.read(option); - VectorizedRowBatch batch = rr.readBatch(100); - - check(batch.size == ROWS_PER_BATCH, - "T6: expected " + ROWS_PER_BATCH + " rows, got " + batch.size); - check(batch.getHiddenColumnVector() == null, - "T6: hiddenColumnVector should be null (exposeHiddenColumn not set)"); - - LongColumnVector xCol = (LongColumnVector) batch.cols[0]; - for (int i = 0; i < batch.size; i++) - { - check(xCol.vector[i] == i * 100L, - "T6: row " + i + " x expected " + (i * 100L) + ", got " + xCol.vector[i]); - } - rr.close(); - reader.close(); - System.out.println("T6 passed!"); - } - - /** - * T7: empty projection + filter + no expose. - * Equivalent to count(*) with timestamp filter. - * filterByHiddenTimestamp=true, needReadHiddenColumn=true, exposeHiddenColumn=false. - * Enters if-branch, filters rows, but hiddenColumnVector stays null. - */ - private static void testT7_EmptyProjection_FilterNoExpose() throws Exception - { - System.out.println("\n--- T7: Empty projection + filter + no expose (filtered count(*)) ---"); - PixelsReader reader = PixelsReaderImpl.newBuilder() - .setStorage(storage).setPath(FILE_WITH_HIDDEN) - .setPixelsFooterCache(new PixelsFooterCache()).build(); - - PixelsReaderOption option = new PixelsReaderOption(); - option.includeCols(new String[0]); - option.transId(0); - option.transTimestamp(FILTER_TIMESTAMP); - option.skipCorruptRecords(true); - option.tolerantSchemaEvolution(true); - - PixelsRecordReader rr = reader.read(option); - VectorizedRowBatch batch = rr.readBatch(TOTAL_ROWS + 100); - - check(batch.cols.length == 0, "T7: expected 0 user columns, got " + batch.cols.length); - check(batch.size == EXPECTED_FILTERED_ROWS, - "T7: expected " + EXPECTED_FILTERED_ROWS + " rows, got " + batch.size); - check(batch.getHiddenColumnVector() == null, - "T7: hiddenColumnVector should be null when exposeHiddenColumn is false"); - rr.close(); - reader.close(); - System.out.println("T7 passed!"); - } - - /** - * T8: exposeHiddenColumn=true on a file without hidden column. - * Constructor must throw IOException. - */ - private static void testT8_ExposeNoHiddenColumn_ThrowsIOException() throws Exception - { - System.out.println("\n--- T8: Expose + no hidden column file → IOException ---"); - PixelsReader reader = PixelsReaderImpl.newBuilder() - .setStorage(storage).setPath(FILE_NO_HIDDEN) - .setPixelsFooterCache(new PixelsFooterCache()).build(); - - PixelsReaderOption option = new PixelsReaderOption(); - option.includeCols(new String[]{"x"}); - option.exposeHiddenColumn(true); - option.skipCorruptRecords(true); - option.tolerantSchemaEvolution(true); - - try - { - reader.read(option); - check(false, "T8: should have thrown IOException"); - } catch (IOException e) - { - check(e.getMessage().contains("no hidden column"), - "T8: exception should mention 'no hidden column', got: " + e.getMessage()); - System.out.println(" Caught expected IOException: " + e.getMessage()); - } - reader.close(); - System.out.println("T8 passed!"); - } - - /** - * T9: filter + no expose (original behavior regression). - * hiddenColumnVector must be null in the result batch. - */ - private static void testT9_FilterNoExpose_HiddenVectorNull() throws Exception - { - System.out.println("\n--- T9: Filter + no expose → hiddenColumnVector null ---"); - PixelsReader reader = PixelsReaderImpl.newBuilder() - .setStorage(storage).setPath(FILE_WITH_HIDDEN) - .setPixelsFooterCache(new PixelsFooterCache()).build(); - - PixelsReaderOption option = new PixelsReaderOption(); - option.includeCols(new String[]{"x", "y"}); - option.transId(0); - option.transTimestamp(FILTER_TIMESTAMP); - option.skipCorruptRecords(true); - option.tolerantSchemaEvolution(true); - // exposeHiddenColumn defaults to false - - PixelsRecordReader rr = reader.read(option); - VectorizedRowBatch batch = rr.readBatch(TOTAL_ROWS + 100); - - check(batch.size == EXPECTED_FILTERED_ROWS, - "T9: expected " + EXPECTED_FILTERED_ROWS + " rows, got " + batch.size); - check(batch.getHiddenColumnVector() == null, - "T9: hiddenColumnVector should be null when exposeHiddenColumn is false"); - - LongColumnVector xCol = (LongColumnVector) batch.cols[0]; - for (int i = 0; i < batch.size; i++) - { - check(xCol.vector[i] == i * 100L, - "T9: row " + i + " x expected " + (i * 100L) + ", got " + xCol.vector[i]); - } - rr.close(); - reader.close(); - System.out.println("T9 passed!"); - } - - /** - * T10: VectorizedRowBatch.applyFilter must sync hiddenColumnVector. - * Read all rows with expose, then apply an external filter keeping even-indexed rows. - */ - private static void testT10_ApplyFilter_SyncsHiddenColumn() throws Exception - { - System.out.println("\n--- T10: applyFilter syncs hiddenColumnVector ---"); - PixelsReader reader = PixelsReaderImpl.newBuilder() - .setStorage(storage).setPath(FILE_WITH_HIDDEN) - .setPixelsFooterCache(new PixelsFooterCache()).build(); - - PixelsReaderOption option = new PixelsReaderOption(); - option.includeCols(new String[]{"x"}); - option.exposeHiddenColumn(true); - option.skipCorruptRecords(true); - option.tolerantSchemaEvolution(true); - - PixelsRecordReader rr = reader.read(option); - VectorizedRowBatch batch = rr.readBatch(TOTAL_ROWS + 100); - check(batch.size == TOTAL_ROWS, "T10: expected " + TOTAL_ROWS + " rows before filter"); - - Bitmap filter = new Bitmap(TOTAL_ROWS, false); - for (int i = 0; i < TOTAL_ROWS; i += 2) - { - filter.set(i); // keep even-indexed rows: 0, 2, 4, ... - } - batch.applyFilter(filter); - - int expectedAfterFilter = (TOTAL_ROWS + 1) / 2; // ceil(30/2) = 15 - check(batch.size == expectedAfterFilter, - "T10: expected " + expectedAfterFilter + " rows after filter, got " + batch.size); - - LongColumnVector xCol = (LongColumnVector) batch.cols[0]; - LongColumnVector hv = batch.getHiddenColumnVector(); - check(hv != null, "T10: hiddenColumnVector should survive applyFilter"); - - for (int i = 0; i < batch.size; i++) - { - int originalIdx = i * 2; - check(xCol.vector[i] == originalIdx * 100L, - "T10: filtered row " + i + " x expected " + (originalIdx * 100L) + ", got " + xCol.vector[i]); - check(hv.vector[i] == expectedTimestamp(originalIdx), - "T10: filtered row " + i + " ts expected " + expectedTimestamp(originalIdx) + ", got " + hv.vector[i]); + recordReader.close(); + System.out.println("Read " + totalRows + " rows from " + pixelsFile); } - rr.close(); - reader.close(); - System.out.println("T10 passed!"); } } diff --git a/pixels-example/src/main/java/io/pixelsdb/pixels/example/core/TestPixelsReaderOption.java b/pixels-example/src/main/java/io/pixelsdb/pixels/example/core/TestPixelsReaderOption.java new file mode 100644 index 0000000000..83ce9bd96d --- /dev/null +++ b/pixels-example/src/main/java/io/pixelsdb/pixels/example/core/TestPixelsReaderOption.java @@ -0,0 +1,165 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the Affero GNU General Public + * License along with Pixels. If not, see + * . + */ +package io.pixelsdb.pixels.example.core; + +import io.pixelsdb.pixels.common.physical.Storage; +import io.pixelsdb.pixels.common.physical.StorageFactory; +import io.pixelsdb.pixels.core.PixelsFooterCache; +import io.pixelsdb.pixels.core.PixelsReader; +import io.pixelsdb.pixels.core.PixelsReaderImpl; +import io.pixelsdb.pixels.core.PixelsWriter; +import io.pixelsdb.pixels.core.PixelsWriterImpl; +import io.pixelsdb.pixels.core.TypeDescription; +import io.pixelsdb.pixels.core.encoding.EncodingLevel; +import io.pixelsdb.pixels.core.reader.PixelsReaderOption; +import io.pixelsdb.pixels.core.reader.PixelsRecordReader; +import io.pixelsdb.pixels.core.vector.IntColumnVector; +import io.pixelsdb.pixels.core.vector.VectorizedRowBatch; + +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Self-contained example for {@link PixelsReaderOption#rgRange(int, int)}. + * It writes four row groups to a temporary local file and verifies whole-file, + * single-row-group, and multi-row-group ranges. + */ +public class TestPixelsReaderOption +{ + private static final String SCHEMA_STRING = "struct"; + private static final int ROWS_PER_GROUP = 8; + private static final int ROW_GROUP_COUNT = 4; + + public static void main(String[] args) throws Exception + { + Storage storage = StorageFactory.Instance().getStorage(Storage.Scheme.file); + Path path = Files.createTempFile("pixels-reader-option-", ".pxl"); + Files.deleteIfExists(path); + + try + { + writeFile(storage, path.toString()); + try (PixelsReader reader = PixelsReaderImpl.newBuilder() + .setStorage(storage) + .setPath(path.toString()) + .setEnableCache(false) + .setPixelsFooterCache(new PixelsFooterCache()) + .build()) + { + check(reader.getRowGroupNum() >= ROW_GROUP_COUNT, + "expected at least " + ROW_GROUP_COUNT + " row groups, got " + + reader.getRowGroupNum()); + + assertRange(reader, 0, ROW_GROUP_COUNT, 0, ROWS_PER_GROUP * ROW_GROUP_COUNT); + assertRange(reader, 0, 1, 0, ROWS_PER_GROUP); + assertRange(reader, 1, 2, ROWS_PER_GROUP, ROWS_PER_GROUP * 2); + assertRange(reader, 2, 2, ROWS_PER_GROUP * 2, ROWS_PER_GROUP * 2); + } + + System.out.println("PixelsReaderOption rgRange example passed."); + } + finally + { + Files.deleteIfExists(path); + } + } + + private static void writeFile(Storage storage, String path) throws Exception + { + TypeDescription schema = TypeDescription.fromString(SCHEMA_STRING); + try (PixelsWriter writer = PixelsWriterImpl.newBuilder() + .setSchema(schema) + .setPixelStride(ROWS_PER_GROUP) + .setRowGroupSize(1) + .setStorage(storage) + .setPath(path) + .setBlockSize(1024 * 1024) + .setReplication((short) 1) + .setBlockPadding(false) + .setOverwrite(true) + .setEncodingLevel(EncodingLevel.EL0) + .setCompressionBlockSize(1) + .setNullsPadding(false) + .build()) + { + for (int group = 0; group < ROW_GROUP_COUNT; ++group) + { + VectorizedRowBatch batch = schema.createRowBatch(ROWS_PER_GROUP); + IntColumnVector x = (IntColumnVector) batch.cols[0]; + IntColumnVector y = (IntColumnVector) batch.cols[1]; + for (int row = 0; row < ROWS_PER_GROUP; ++row) + { + int globalRow = group * ROWS_PER_GROUP + row; + x.add(globalRow); + y.add(globalRow * 10); + } + batch.size = ROWS_PER_GROUP; + writer.addRowBatch(batch); + } + } + } + + private static void assertRange(PixelsReader reader, int rgStart, int rgLen, + int expectedStart, int expectedRows) throws Exception + { + PixelsReaderOption option = new PixelsReaderOption() + .includeCols(new String[]{"x", "y"}) + .rgRange(rgStart, rgLen) + .skipCorruptRecords(true) + .tolerantSchemaEvolution(true); + + int rowsRead = 0; + try (PixelsRecordReader recordReader = reader.read(option)) + { + while (true) + { + VectorizedRowBatch batch = recordReader.readBatch(7); + IntColumnVector x = (IntColumnVector) batch.cols[0]; + IntColumnVector y = (IntColumnVector) batch.cols[1]; + for (int row = 0; row < batch.size; ++row) + { + int expected = expectedStart + rowsRead; + check(x.vector[row] == expected, + "x mismatch for rgRange(" + rgStart + ", " + rgLen + ") at row " + + rowsRead + ": expected " + expected + ", got " + x.vector[row]); + check(y.vector[row] == expected * 10, + "y mismatch for rgRange(" + rgStart + ", " + rgLen + ") at row " + + rowsRead + ": expected " + expected * 10 + ", got " + y.vector[row]); + rowsRead++; + } + if (batch.endOfFile) + { + break; + } + } + } + check(rowsRead == expectedRows, + "row count mismatch for rgRange(" + rgStart + ", " + rgLen + "): expected " + + expectedRows + ", got " + rowsRead); + } + + private static void check(boolean condition, String message) + { + if (!condition) + { + throw new AssertionError(message); + } + } +} diff --git a/pixels-example/src/main/java/io/pixelsdb/pixels/example/core/TestPixelsWriter.java b/pixels-example/src/main/java/io/pixelsdb/pixels/example/core/TestPixelsWriter.java index 5f5ef728d9..359d1ecf17 100644 --- a/pixels-example/src/main/java/io/pixelsdb/pixels/example/core/TestPixelsWriter.java +++ b/pixels-example/src/main/java/io/pixelsdb/pixels/example/core/TestPixelsWriter.java @@ -25,87 +25,69 @@ import io.pixelsdb.pixels.core.PixelsWriterImpl; import io.pixelsdb.pixels.core.TypeDescription; import io.pixelsdb.pixels.core.encoding.EncodingLevel; -import io.pixelsdb.pixels.core.exception.PixelsWriterException; -import io.pixelsdb.pixels.core.vector.*; - -import java.io.IOException; -import java.sql.Timestamp; +import io.pixelsdb.pixels.core.vector.BinaryColumnVector; +import io.pixelsdb.pixels.core.vector.IntColumnVector; +import io.pixelsdb.pixels.core.vector.VectorizedRowBatch; /** + * Minimal example showing how to write a Pixels file to the local file system. + *

+ * The flow is: build a {@link PixelsWriter} against a {@link TypeDescription} schema, + * fill {@link VectorizedRowBatch}es column by column, add them to the writer, and close. + *

+ * For an end-to-end write-then-read example with per-value verification across all + * column types, see {@code TestPixelsReadWrite}. + * * @author hank * @create 2018-11-19 */ public class TestPixelsWriter { - public static void main(String[] args) throws IOException - { - // Note you may need to restart intellij to let it pick up the updated environment variable value - // example path: s3://bucket-name/test-file.pxl - String pixelsFile = System.getenv("PIXELS_S3_TEST_BUCKET_PATH") + "test.pxl"; - Storage storage = StorageFactory.Instance().getStorage("s3"); + private static final String SCHEMA_STRING = "struct"; + private static final int ROW_NUM = 1000; - String schemaStr = "struct"; + public static void main(String[] args) throws Exception + { + String pixelsFile = "/tmp/pixels-writer-example.pxl"; + Storage storage = StorageFactory.Instance().getStorage(Storage.Scheme.file); - try - { - TypeDescription schema = TypeDescription.fromString(schemaStr); - VectorizedRowBatch rowBatch = schema.createRowBatch(); - LongColumnVector a = (LongColumnVector) rowBatch.cols[0]; // int - FloatColumnVector b = (FloatColumnVector) rowBatch.cols[1]; // float - DoubleColumnVector c = (DoubleColumnVector) rowBatch.cols[2]; // double - TimestampColumnVector d = (TimestampColumnVector) rowBatch.cols[3]; // timestamp - ByteColumnVector e = (ByteColumnVector) rowBatch.cols[4]; // boolean - BinaryColumnVector z = (BinaryColumnVector) rowBatch.cols[5]; // string + TypeDescription schema = TypeDescription.fromString(SCHEMA_STRING); + PixelsWriter writer = PixelsWriterImpl.newBuilder() + .setSchema(schema) + .setPixelStride(10000) + .setRowGroupSize(64 * 1024 * 1024) + .setStorage(storage) + .setPath(pixelsFile) + .setBlockSize(256 * 1024 * 1024) + .setReplication((short) 1) + .setBlockPadding(true) + .setEncodingLevel(EncodingLevel.EL2) + .setCompressionBlockSize(1) + .build(); - PixelsWriter pixelsWriter = - PixelsWriterImpl.newBuilder() - .setSchema(schema) - .setPixelStride(10000) - .setRowGroupSize(64 * 1024 * 1024) - .setStorage(storage) - .setPath(pixelsFile) - .setBlockSize(256 * 1024 * 1024) - .setReplication((short) 3) - .setBlockPadding(true) - .setEncodingLevel(EncodingLevel.EL2) - .setCompressionBlockSize(1) - .build(); + VectorizedRowBatch rowBatch = schema.createRowBatch(); + IntColumnVector id = (IntColumnVector) rowBatch.cols[0]; + BinaryColumnVector name = (BinaryColumnVector) rowBatch.cols[1]; - long curT = System.currentTimeMillis(); - Timestamp timestamp = new Timestamp(curT); - for (int i = 0; i < 1; i++) - { - int row = rowBatch.size++; - a.vector[row] = i; - a.isNull[row] = false; - b.vector[row] = Float.floatToIntBits(i * 3.1415f); - b.isNull[row] = false; - c.vector[row] = Double.doubleToLongBits(i * 3.14159d); - c.isNull[row] = false; - d.set(row, timestamp); - d.isNull[row] = false; - e.vector[row] = (byte) (i > 25000 ? 1 : 0); - e.isNull[row] = false; - z.setVal(row, String.valueOf(i).getBytes()); - z.isNull[row] = false; - if (rowBatch.size == rowBatch.getMaxSize()) - { - pixelsWriter.addRowBatch(rowBatch); - rowBatch.reset(); - } - } - - if (rowBatch.size != 0) + for (int i = 0; i < ROW_NUM; i++) + { + int row = rowBatch.size++; + id.vector[row] = i; + id.isNull[row] = false; + name.setVal(row, ("row-" + i).getBytes()); + name.isNull[row] = false; + if (rowBatch.size == rowBatch.getMaxSize()) { - pixelsWriter.addRowBatch(rowBatch); - System.out.println("A rowBatch of size " + rowBatch.size + " has been written to " + pixelsFile); + writer.addRowBatch(rowBatch); rowBatch.reset(); } - - pixelsWriter.close(); - } catch (IOException | PixelsWriterException e) + } + if (rowBatch.size != 0) { - e.printStackTrace(); + writer.addRowBatch(rowBatch); + rowBatch.reset(); } + writer.close(); + System.out.println("Written " + ROW_NUM + " rows to " + pixelsFile); } } diff --git a/pixels-executor/src/main/java/io/pixelsdb/pixels/executor/aggregation/Aggregator.java b/pixels-executor/src/main/java/io/pixelsdb/pixels/executor/aggregation/Aggregator.java index 506e385ae3..195e7f9ed2 100644 --- a/pixels-executor/src/main/java/io/pixelsdb/pixels/executor/aggregation/Aggregator.java +++ b/pixels-executor/src/main/java/io/pixelsdb/pixels/executor/aggregation/Aggregator.java @@ -192,7 +192,7 @@ public void aggregate(VectorizedRowBatch inputRowBatch) public boolean writeAggrOutput(PixelsWriter pixelsWriter) throws IOException { - VectorizedRowBatch outputRowBatch = this.outputSchema.createRowBatch(this.batchSize, TypeDescription.Mode.NONE); + VectorizedRowBatch outputRowBatch = this.outputSchema.createRowBatch(this.batchSize); if (partition) { for (int hash = 0; hash < this.numPartitions; ++hash) diff --git a/pixels-executor/src/main/java/io/pixelsdb/pixels/executor/aggregation/function/BigintSum.java b/pixels-executor/src/main/java/io/pixelsdb/pixels/executor/aggregation/function/BigintSum.java index cf53e0756b..59fc45b117 100644 --- a/pixels-executor/src/main/java/io/pixelsdb/pixels/executor/aggregation/function/BigintSum.java +++ b/pixels-executor/src/main/java/io/pixelsdb/pixels/executor/aggregation/function/BigintSum.java @@ -20,7 +20,9 @@ package io.pixelsdb.pixels.executor.aggregation.function; import io.pixelsdb.pixels.core.vector.ColumnVector; +import io.pixelsdb.pixels.core.vector.IntColumnVector; import io.pixelsdb.pixels.core.vector.LongColumnVector; +import io.pixelsdb.pixels.core.vector.ShortColumnVector; /** * @author hank @@ -37,8 +39,23 @@ public void input(int rowId, ColumnVector inputVector) { if (inputVector.noNulls || !inputVector.isNull[rowId]) { - LongColumnVector longColumnVector = (LongColumnVector) inputVector; - this.value += longColumnVector.vector[rowId]; + if (inputVector instanceof ShortColumnVector) + { + this.value += ((ShortColumnVector) inputVector).vector[rowId]; + } + else if (inputVector instanceof IntColumnVector) + { + this.value += ((IntColumnVector) inputVector).vector[rowId]; + } + else if (inputVector instanceof LongColumnVector) + { + this.value += ((LongColumnVector) inputVector).vector[rowId]; + } + else + { + throw new IllegalStateException("Unsupported input vector: " + + inputVector.getClass().getName()); + } } } diff --git a/pixels-executor/src/main/java/io/pixelsdb/pixels/executor/join/HashJoiner.java b/pixels-executor/src/main/java/io/pixelsdb/pixels/executor/join/HashJoiner.java index 8d0450745b..0d10a9c851 100644 --- a/pixels-executor/src/main/java/io/pixelsdb/pixels/executor/join/HashJoiner.java +++ b/pixels-executor/src/main/java/io/pixelsdb/pixels/executor/join/HashJoiner.java @@ -68,7 +68,7 @@ public List join(VectorizedRowBatch largeBatch) requireNonNull(largeBatch, "largeBatch is null"); checkArgument(largeBatch.size > 0, "largeBatch is empty"); List result = new LinkedList<>(); - VectorizedRowBatch joinedRowBatch = this.joinedSchema.createRowBatch(largeBatch.maxSize, TypeDescription.Mode.NONE); + VectorizedRowBatch joinedRowBatch = this.joinedSchema.createRowBatch(largeBatch.maxSize); Tuple.Builder builder = new Tuple.Builder(largeBatch, this.largeKeyColumnIds, this.largeProjection); while (builder.hasNext()) { @@ -87,7 +87,7 @@ public List join(VectorizedRowBatch largeBatch) if (joinedRowBatch.isFull()) { result.add(joinedRowBatch); - joinedRowBatch = this.joinedSchema.createRowBatch(largeBatch.maxSize, TypeDescription.Mode.NONE); + joinedRowBatch = this.joinedSchema.createRowBatch(largeBatch.maxSize); } joined.writeTo(joinedRowBatch); break; @@ -111,7 +111,7 @@ public List join(VectorizedRowBatch largeBatch) if (joinedRowBatch.isFull()) { result.add(joinedRowBatch); - joinedRowBatch = this.joinedSchema.createRowBatch(largeBatch.maxSize, TypeDescription.Mode.NONE); + joinedRowBatch = this.joinedSchema.createRowBatch(largeBatch.maxSize); } joined.writeTo(joinedRowBatch); smallHead = smallHead.next; @@ -141,7 +141,7 @@ public boolean writeLeftOuter(PixelsWriter pixelsWriter, int batchSize) throws I leftOuterTuples.add(small); } } - VectorizedRowBatch leftOuterBatch = this.joinedSchema.createRowBatch(batchSize, TypeDescription.Mode.NONE); + VectorizedRowBatch leftOuterBatch = this.joinedSchema.createRowBatch(batchSize); for (Tuple small : leftOuterTuples) { if (leftOuterBatch.isFull()) @@ -187,7 +187,7 @@ public boolean writeLeftOuterAndPartition(PixelsWriter pixelsWriter, final int b leftOuterTuples.add(small); } } - VectorizedRowBatch leftOuterBatch = this.joinedSchema.createRowBatch(batchSize, TypeDescription.Mode.NONE); + VectorizedRowBatch leftOuterBatch = this.joinedSchema.createRowBatch(batchSize); for (Tuple small : leftOuterTuples) { if (leftOuterBatch.isFull()) diff --git a/pixels-executor/src/main/java/io/pixelsdb/pixels/executor/join/Partitioner.java b/pixels-executor/src/main/java/io/pixelsdb/pixels/executor/join/Partitioner.java index 135c2b01b5..0814cb327b 100644 --- a/pixels-executor/src/main/java/io/pixelsdb/pixels/executor/join/Partitioner.java +++ b/pixels-executor/src/main/java/io/pixelsdb/pixels/executor/join/Partitioner.java @@ -83,7 +83,7 @@ public Partitioner(int numPartition, int batchSize, TypeDescription schema, int[ this.selectedArrayIndexes = new int[numPartition]; for (int i = 0; i < numPartition; ++i) { - this.rowBatches[i] = schema.createRowBatch(batchSize, TypeDescription.Mode.NONE); + this.rowBatches[i] = schema.createRowBatch(batchSize); this.selectedArrays[i] = new int[batchSize]; this.selectedArrayIndexes[i] = 0; } @@ -127,14 +127,14 @@ public Map partition(VectorizedRowBatch input) if (freeSlots == 0) { output.put(hash, rowBatches[hash]); - rowBatches[hash] = schema.createRowBatch(batchSize, TypeDescription.Mode.NONE); + rowBatches[hash] = schema.createRowBatch(batchSize); rowBatches[hash].addSelected(selected, 0, selectedLength, input); } else if (freeSlots <= selectedLength) { rowBatches[hash].addSelected(selected, 0, freeSlots, input); output.put(hash, rowBatches[hash]); - rowBatches[hash] = schema.createRowBatch(batchSize, TypeDescription.Mode.NONE); + rowBatches[hash] = schema.createRowBatch(batchSize); if (freeSlots < selectedLength) { rowBatches[hash].addSelected(selected, freeSlots, selectedLength - freeSlots, input); diff --git a/pixels-executor/src/main/java/io/pixelsdb/pixels/executor/predicate/ColumnFilter.java b/pixels-executor/src/main/java/io/pixelsdb/pixels/executor/predicate/ColumnFilter.java index 8f861ccc9a..5c2324490a 100644 --- a/pixels-executor/src/main/java/io/pixelsdb/pixels/executor/predicate/ColumnFilter.java +++ b/pixels-executor/src/main/java/io/pixelsdb/pixels/executor/predicate/ColumnFilter.java @@ -97,6 +97,10 @@ public ColumnFilter(String columnName, TypeDescription.Category columnType, Stri { filterType = new TypeReference>(){}.getType(); } + else if (columnJavaType == short.class) + { + filterType = new TypeReference>(){}.getType(); + } else if (columnJavaType == int.class) { filterType = new TypeReference>(){}.getType(); @@ -316,7 +320,13 @@ public void doFilter(ColumnVector columnVector, int start, int length, Bitmap re doFilter(bcv.vector, bcv.noNulls ? null : bcv.isNull, start, length, result); return; case SHORT: + ShortColumnVector scv = (ShortColumnVector) columnVector; + doFilter(scv.vector, scv.noNulls ? null : scv.isNull, start, length, result); + return; case INT: + IntColumnVector icv = (IntColumnVector) columnVector; + doFilter(icv.vector, icv.noNulls ? null : icv.isNull, start, length, result); + return; case LONG: LongColumnVector lcv = (LongColumnVector) columnVector; doFilter(lcv.vector, lcv.noNulls ? null : lcv.isNull, start, length, result); @@ -473,6 +483,98 @@ private void doFilter(byte[] vector, boolean[] isNull, int start, int length, Bi } } + private void doFilter(short[] vector, boolean[] isNull, int start, int length, Bitmap result) + { + boolean noNulls = isNull == null; + if (!this.filter.ranges.isEmpty()) + { + for (Range range : this.filter.ranges) + { + short lowerBound = range.lowerBound.type != Bound.Type.UNBOUNDED ? + (Short) range.lowerBound.value : Short.MIN_VALUE; + if (range.lowerBound.type == Bound.Type.EXCLUDED) + { + lowerBound++; + } + short upperBound = range.upperBound.type != Bound.Type.UNBOUNDED ? + (Short) range.upperBound.value : Short.MAX_VALUE; + if (range.upperBound.type == Bound.Type.EXCLUDED) + { + upperBound--; + } + if (this.filter.allowNull && !noNulls) + { + for (int i = start; i < start + length; ++i) + { + if (isNull[i] || vector[i] >= lowerBound && vector[i] <= upperBound) + { + result.set(i); + } + } + } + else + { + for (int i = start; i < start + length; ++i) + { + if ((noNulls || !isNull[i]) && vector[i] >= lowerBound && vector[i] <= upperBound) + { + result.set(i); + } + } + } + } + } + else + { + if (!includes.isEmpty()) + { + if (this.filter.allowNull && !noNulls) + { + for (int i = start; i < start + length; ++i) + { + if (isNull[i] || includes.contains(vector[i])) + { + result.set(i); + } + } + } + else + { + for (int i = start; i < start + length; ++i) + { + if ((noNulls || !isNull[i]) && includes.contains(vector[i])) + { + result.set(i); + } + } + } + } + if (!excludes.isEmpty()) + { + if (this.filter.allowNull && !noNulls) + { + for (int i = start; i < start + length; ++i) + { + if (isNull[i] || !excludes.contains(vector[i])) + { + result.set(i); + } + } + } + else + { + for (int i = start; i < start + length; ++i) + { + if ((noNulls || !isNull[i]) && !excludes.contains(vector[i])) + { + result.set(i); + } + } + } + } + } + } + private void doFilter(long[] vector, boolean[] isNull, int start, int length, Bitmap result) { boolean noNulls = isNull == null; diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/MemTable.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/MemTable.java index cefa83c90f..18b6f839a5 100644 --- a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/MemTable.java +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/MemTable.java @@ -39,12 +39,12 @@ public class MemTable implements Referenceable private final int startIndex; private final int length; - public MemTable(long id, TypeDescription schema, int size, int mode, + public MemTable(long id, TypeDescription schema, int size, long fileId, int startIndex, int length) { this.id = id; this.schema = schema; - this.rowBatch = schema.createRowBatchWithHiddenColumn(size, mode); + this.rowBatch = schema.createRowBatchWithHiddenColumn(size); this.fileId = fileId; this.startIndex = startIndex; this.length = length; diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/PixelsWriteBuffer.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/PixelsWriteBuffer.java index 799e487cbf..7c264e0de5 100644 --- a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/PixelsWriteBuffer.java +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/PixelsWriteBuffer.java @@ -181,8 +181,7 @@ public PixelsWriteBuffer(long tableId, TypeDescription schema, Path targetOrdere idCounter, this.memTableSize * this.maxMemTableCount, retinaHostName, virtualNodeId); this.ingestFilePublisher = new IngestFilePublisher(this.currentFileWriterManager.getFirstBlockId()); - this.activeMemTable = new MemTable(this.idCounter, schema, memTableSize, - TypeDescription.Mode.CREATE_INT_VECTOR_FOR_INT, this.currentFileWriterManager.getFileId(), + this.activeMemTable = new MemTable(this.idCounter, schema, memTableSize, this.currentFileWriterManager.getFileId(), 0, this.memTableSize); this.idCounter++; this.currentMemTableCount = 1; @@ -298,9 +297,7 @@ private void retireActiveMemTableLocked() throws RetinaException MemTable oldMemTable = this.activeMemTable; SuperVersion oldVersion = this.currentVersion; this.immutableMemTables.add(this.activeMemTable); - this.activeMemTable = new MemTable(this.idCounter, this.schema, - this.memTableSize, TypeDescription.Mode.CREATE_INT_VECTOR_FOR_INT, - this.currentFileWriterManager.getFileId(), + this.activeMemTable = new MemTable(this.idCounter, this.schema, this.memTableSize, this.currentFileWriterManager.getFileId(), this.currentMemTableCount * this.memTableSize, this.memTableSize); this.currentMemTableCount += 1; diff --git a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestPixelsWriteBuffer.java b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestPixelsWriteBuffer.java index 4eb9a0dd08..12b924af2a 100644 --- a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestPixelsWriteBuffer.java +++ b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestPixelsWriteBuffer.java @@ -154,8 +154,7 @@ private static MemTable newMemTable(int size) { TypeDescription schema = TypeDescription.createSchemaFromStrings( Arrays.asList("id"), Arrays.asList("int")); - return new MemTable(0L, schema, size, - TypeDescription.Mode.CREATE_INT_VECTOR_FOR_INT, 100L, 0, size); + return new MemTable(0L, schema, size, 100L, 0, size); } private static byte[][] row(int value) diff --git a/pixels-turbo/pixels-invoker-lambda/src/test/java/io/pixelsdb/pixels/invoker/lambda/TestBroadcastJoinLambdaInvoker.java b/pixels-turbo/pixels-invoker-lambda/src/test/java/io/pixelsdb/pixels/invoker/lambda/TestBroadcastJoinLambdaInvoker.java index 87ad3ccfe3..227a408323 100644 --- a/pixels-turbo/pixels-invoker-lambda/src/test/java/io/pixelsdb/pixels/invoker/lambda/TestBroadcastJoinLambdaInvoker.java +++ b/pixels-turbo/pixels-invoker-lambda/src/test/java/io/pixelsdb/pixels/invoker/lambda/TestBroadcastJoinLambdaInvoker.java @@ -120,17 +120,17 @@ public void testPartLineitem() throws ExecutionException, InterruptedException @Test public void testSerFilter() { - ArrayList> discreteValues = new ArrayList<>(); - discreteValues.add(new Bound<>(INCLUDED, 49L)); - discreteValues.add(new Bound<>(INCLUDED, 14L)); - discreteValues.add(new Bound<>(INCLUDED, 23L)); - discreteValues.add(new Bound<>(INCLUDED, 45L)); - discreteValues.add(new Bound<>(INCLUDED, 19L)); - discreteValues.add(new Bound<>(INCLUDED, 3L)); - discreteValues.add(new Bound<>(INCLUDED, 36L)); - discreteValues.add(new Bound<>(INCLUDED, 9L)); - ColumnFilter columnFilter = new ColumnFilter("p_size", TypeDescription.Category.INT, - new Filter<>(Long.TYPE, new ArrayList<>(), discreteValues, false, false, false, false)); + ArrayList> discreteValues = new ArrayList<>(); + discreteValues.add(new Bound<>(INCLUDED, 49)); + discreteValues.add(new Bound<>(INCLUDED, 14)); + discreteValues.add(new Bound<>(INCLUDED, 23)); + discreteValues.add(new Bound<>(INCLUDED, 45)); + discreteValues.add(new Bound<>(INCLUDED, 19)); + discreteValues.add(new Bound<>(INCLUDED, 3)); + discreteValues.add(new Bound<>(INCLUDED, 36)); + discreteValues.add(new Bound<>(INCLUDED, 9)); + ColumnFilter columnFilter = new ColumnFilter<>("p_size", TypeDescription.Category.INT, + new Filter<>(Integer.TYPE, new ArrayList<>(), discreteValues, false, false, false, false)); SortedMap columnFilters = new TreeMap<>(); columnFilters.put(2, columnFilter); TableScanFilter filter = new TableScanFilter("tpch", "lineitem", columnFilters); @@ -142,7 +142,7 @@ public void testDeFilter() { String filter = "{\"schemaName\":\"tpch\",\"tableName\":\"lineitem\"," + "\"columnFilters\":{2:{\"columnName\":\"p_size\",\"columnType\":\"INT\"," + - "\"filterJson\":\"{\\\"javaType\\\":\\\"long\\\",\\\"isAll\\\":false," + + "\"filterJson\":\"{\\\"javaType\\\":\\\"int\\\",\\\"isAll\\\":false," + "\\\"isNone\\\":false,\\\"allowNull\\\":false,\\\"onlyNull\\\":false," + "\\\"ranges\\\":[],\\\"discreteValues\\\":[{" + "\\\"type\\\":\\\"INCLUDED\\\",\\\"value\\\":49}," +