From b72646d5f6ad5b068004d7264be48e5706a53ebe Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Tue, 11 Aug 2026 16:47:19 +0800 Subject: [PATCH 1/2] fix: retina write path to remap RPC values into layout column order --- .../io/pixelsdb/pixels/retina/MemTable.java | 13 ++++++---- .../pixels/retina/PixelsWriteBuffer.java | 18 +++++++++---- .../pixels/retina/RetinaResourceManager.java | 26 ++++++++++++++++--- .../pixels/retina/TestPixelsWriteBuffer.java | 13 ++++++++-- 4 files changed, 54 insertions(+), 16 deletions(-) diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/MemTable.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/MemTable.java index 4f9c0d1b4..b58dfd174 100644 --- a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/MemTable.java +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/MemTable.java @@ -33,18 +33,20 @@ public class MemTable implements Referenceable private final long id; // unique identifier private final TypeDescription schema; private final VectorizedRowBatch rowBatch; + private final int[] orderMapping; private final long fileId; private final int startIndex; private final int length; - public MemTable(long id, TypeDescription schema, int size, - long fileId, int startIndex, int length) + public MemTable(long id, TypeDescription schema, int size, int vectorLayout, + int[] orderMapping, long fileId, int startIndex, int length) { this.id = id; this.schema = schema; - this.rowBatch = schema.createRowBatchWithHiddenColumn(size); + this.orderMapping = orderMapping; + this.rowBatch = schema.createRowBatchWithHiddenColumn(size, vectorLayout); this.fileId = fileId; this.startIndex = startIndex; this.length = length; @@ -68,13 +70,14 @@ public synchronized int add(byte[][] values, long timestamp) throws RetinaExcept } for (int i = 0; i < values.length; ++i) { - if (values[i] == null) + byte[] value = values[this.orderMapping[i]]; + if (value == null) { this.rowBatch.cols[i].addNull(); } else { - this.rowBatch.cols[i].add(values[i]); + this.rowBatch.cols[i].add(value); } } this.rowBatch.cols[schema.getChildren().size()].add(timestamp); diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/PixelsWriteBuffer.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/PixelsWriteBuffer.java index 7c264e0de..d694a5b84 100644 --- a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/PixelsWriteBuffer.java +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/PixelsWriteBuffer.java @@ -69,6 +69,9 @@ public class PixelsWriteBuffer // column information is recorded to create rowBatch private final TypeDescription schema; + // column order mapping from layout to metadata column order + private final int[] orderMapping; + // configuration information of PixelsWriter private final int memTableSize; private final long blockSize; @@ -130,11 +133,13 @@ public class PixelsWriteBuffer private final int virtualNodeId; private final IndexOption indexOption; - public PixelsWriteBuffer(long tableId, TypeDescription schema, Path targetOrderedDirPath, - Path targetCompactDirPath, String retinaHostName, int virtualNode) throws RetinaException + public PixelsWriteBuffer(long tableId, TypeDescription schema, int[] orderMapping, + Path targetOrderedDirPath, Path targetCompactDirPath, + String retinaHostName, int virtualNode) throws RetinaException { this.tableId = tableId; this.schema = schema; + this.orderMapping = orderMapping; this.virtualNodeId = virtualNode; this.indexOption = IndexOption.builder() .vNodeId(virtualNodeId) @@ -181,8 +186,9 @@ public PixelsWriteBuffer(long tableId, TypeDescription schema, Path targetOrdere idCounter, this.memTableSize * this.maxMemTableCount, retinaHostName, virtualNodeId); this.ingestFilePublisher = new IngestFilePublisher(this.currentFileWriterManager.getFirstBlockId()); - this.activeMemTable = new MemTable(this.idCounter, schema, memTableSize, this.currentFileWriterManager.getFileId(), - 0, this.memTableSize); + this.activeMemTable = new MemTable(this.idCounter, schema, memTableSize, + TypeDescription.VectorLayout.NONE, this.orderMapping, + this.currentFileWriterManager.getFileId(), 0, this.memTableSize); this.idCounter++; this.currentMemTableCount = 1; @@ -297,7 +303,9 @@ private void retireActiveMemTableLocked() throws RetinaException MemTable oldMemTable = this.activeMemTable; SuperVersion oldVersion = this.currentVersion; this.immutableMemTables.add(this.activeMemTable); - this.activeMemTable = new MemTable(this.idCounter, this.schema, this.memTableSize, this.currentFileWriterManager.getFileId(), + this.activeMemTable = new MemTable(this.idCounter, this.schema, + this.memTableSize, TypeDescription.VectorLayout.NONE, + this.orderMapping, this.currentFileWriterManager.getFileId(), this.currentMemTableCount * this.memTableSize, this.memTableSize); this.currentMemTableCount += 1; diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RetinaResourceManager.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RetinaResourceManager.java index d22563b0e..f06ab3c08 100644 --- a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RetinaResourceManager.java +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RetinaResourceManager.java @@ -680,10 +680,28 @@ public void addWriteBuffer(String schemaName, String tableName) throws RetinaExc List orderedPaths = latestLayout.getOrderedPaths(); List compactPaths = latestLayout.getCompactPaths(); - // get schema + // Build the file schema in layout order while retaining a mapping from + // layout positions to the metadata order used by Retina RPC values. List columns = this.metadataService.getColumns(schemaName, tableName, false); - List columnNames = columns.stream().map(Column::getName).collect(Collectors.toList()); - List columnTypes = columns.stream().map(Column::getType).collect(Collectors.toList()); + List layoutColumnOrder = latestLayout.getOrdered().getColumnOrder(); + List metadataColumnNames = new ArrayList<>(columns.size()); + for (Column column : columns) + { + metadataColumnNames.add(column.getName()); + } + int[] orderMapping = new int[layoutColumnOrder.size()]; + for (int i = 0; i < layoutColumnOrder.size(); ++i) + { + orderMapping[i] = metadataColumnNames.indexOf(layoutColumnOrder.get(i)); + } + List columnNames = new ArrayList<>(columns.size()); + List columnTypes = new ArrayList<>(columns.size()); + for (int metadataColumnIndex : orderMapping) + { + Column column = columns.get(metadataColumnIndex); + columnNames.add(column.getName()); + columnTypes.add(column.getType()); + } TypeDescription schema = TypeDescription.createSchemaFromStrings(columnNames, columnTypes); String writeBufferKey = RetinaUtils.buildWriteBufferKey(schemaName, tableName); @@ -693,7 +711,7 @@ public void addWriteBuffer(String schemaName, String tableName) throws RetinaExc for (int i = 0; i < totalVirtualNodeNum; i++) { PixelsWriteBuffer pixelsWriteBuffer = new PixelsWriteBuffer(latestLayout.getTableId(), - schema, orderedPaths.get(0), compactPaths.get(0), retinaHostName, i); + schema, orderMapping, orderedPaths.get(0), compactPaths.get(0), retinaHostName, i); nodeBuffers.put(i, pixelsWriteBuffer); } } catch (Exception e) diff --git a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestPixelsWriteBuffer.java b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestPixelsWriteBuffer.java index 12b924af2..4215a2799 100644 --- a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestPixelsWriteBuffer.java +++ b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestPixelsWriteBuffer.java @@ -69,7 +69,14 @@ public void testConcurrentWriteOperations() { try { - buffer = new PixelsWriteBuffer(0L, schema, targetOrderDirPath, targetCompactDirPath, "localhost", 0); // table id get from mysql `TBLS` table + int[] orderMapping = new int[schema.getChildren().size()]; + for (int i = 0; i < orderMapping.length; ++i) + { + orderMapping[i] = i; + } + // table id get from mysql `TBLS` table + buffer = new PixelsWriteBuffer(0L, schema, orderMapping, targetOrderDirPath, + targetCompactDirPath, "localhost", 0); } catch (Exception e) { System.out.println("setup error: " + e); @@ -154,7 +161,9 @@ private static MemTable newMemTable(int size) { TypeDescription schema = TypeDescription.createSchemaFromStrings( Arrays.asList("id"), Arrays.asList("int")); - return new MemTable(0L, schema, size, 100L, 0, size); + int[] orderMapping = new int[] {0}; + return new MemTable(0L, schema, size, TypeDescription.VectorLayout.NONE, + orderMapping, 100L, 0, size); } private static byte[][] row(int value) From e43c991ed6a5c17723f88299e87bbf7b42620393 Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Tue, 11 Aug 2026 16:47:28 +0800 Subject: [PATCH 2/2] fix: STAT to collect regular files from metadata paths --- .../pixels/cli/executor/StatExecutor.java | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/StatExecutor.java b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/StatExecutor.java index 4fb160106..3cc7ad373 100644 --- a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/StatExecutor.java +++ b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/StatExecutor.java @@ -24,7 +24,9 @@ import io.pixelsdb.pixels.cli.Main; import io.pixelsdb.pixels.common.metadata.MetadataService; import io.pixelsdb.pixels.common.metadata.domain.Column; +import io.pixelsdb.pixels.common.metadata.domain.File; import io.pixelsdb.pixels.common.metadata.domain.Layout; +import io.pixelsdb.pixels.common.metadata.domain.Path; import io.pixelsdb.pixels.common.physical.Storage; import io.pixelsdb.pixels.common.physical.StorageFactory; import io.pixelsdb.pixels.common.utils.ConfigFactory; @@ -72,19 +74,35 @@ public void execute(Namespace ns, String command) throws Exception { String[] orderedPaths = layout.getOrderedPathUris(); Main.validateOrderedOrCompactPaths(orderedPaths); - Storage storage = StorageFactory.Instance().getStorage(orderedPaths[0]); - files.addAll(storage.listPaths(orderedPaths)); + for (Path path : layout.getOrderedPaths()) + { + for (File file : metadataService.getRegularFiles(path.getId())) + { + files.add(File.getFilePath(path, file)); + } + } } if (compactEnabled) { String[] compactPaths = layout.getCompactPathUris(); Main.validateOrderedOrCompactPaths(compactPaths); - Storage storage = StorageFactory.Instance().getStorage(compactPaths[0]); - files.addAll(storage.listPaths(compactPaths)); + for (Path path : layout.getCompactPaths()) + { + for (File file : metadataService.getRegularFiles(path.getId())) + { + files.add(File.getFilePath(path, file)); + } + } } } } + if (files.isEmpty()) + { + System.out.println("No regular files found; statistics were not updated."); + return; + } + // get the statistics. long startTime = System.currentTimeMillis();