[Issue #1376] fix Retina layout column remapping and STAT regular-file listing

pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/StatExecutor.java

@@ -24,7 +24,9 @@
 import io.pixelsdb.pixels.cli.Main;
 import io.pixelsdb.pixels.common.metadata.MetadataService;
 import io.pixelsdb.pixels.common.metadata.domain.Column;
+import io.pixelsdb.pixels.common.metadata.domain.File;
 import io.pixelsdb.pixels.common.metadata.domain.Layout;
+import io.pixelsdb.pixels.common.metadata.domain.Path;
 import io.pixelsdb.pixels.common.physical.Storage;
 import io.pixelsdb.pixels.common.physical.StorageFactory;
 import io.pixelsdb.pixels.common.utils.ConfigFactory;
@@ -72,19 +74,35 @@ public void execute(Namespace ns, String command) throws Exception
                 {
                     String[] orderedPaths = layout.getOrderedPathUris();
                     Main.validateOrderedOrCompactPaths(orderedPaths);
-                    Storage storage = StorageFactory.Instance().getStorage(orderedPaths[0]);
-                    files.addAll(storage.listPaths(orderedPaths));
+                    for (Path path : layout.getOrderedPaths())
+                    {
+                        for (File file : metadataService.getRegularFiles(path.getId()))
+                        {
+                            files.add(File.getFilePath(path, file));
+                        }
+                    }
                 }
                 if (compactEnabled)
                 {
                     String[] compactPaths = layout.getCompactPathUris();
                     Main.validateOrderedOrCompactPaths(compactPaths);
-                    Storage storage = StorageFactory.Instance().getStorage(compactPaths[0]);
-                    files.addAll(storage.listPaths(compactPaths));
+                    for (Path path : layout.getCompactPaths())
+                    {
+                        for (File file : metadataService.getRegularFiles(path.getId()))
+                        {
+                            files.add(File.getFilePath(path, file));
+                        }
+                    }
                 }
             }
         }
 
+        if (files.isEmpty())
+        {
+            System.out.println("No regular files found; statistics were not updated.");
+            return;
+        }
+
         // get the statistics.
         long startTime = System.currentTimeMillis();
 

pixels-retina/src/main/java/io/pixelsdb/pixels/retina/MemTable.java

@@ -33,18 +33,20 @@ public class MemTable implements Referenceable
     private final long id;  // unique identifier
     private final TypeDescription schema;
     private final VectorizedRowBatch rowBatch;
+    private final int[] orderMapping;
 
 
     private final long fileId;
     private final int startIndex;
     private final int length;
 
-    public MemTable(long id, TypeDescription schema, int size,
-                    long fileId, int startIndex, int length)
+    public MemTable(long id, TypeDescription schema, int size, int vectorLayout,
+                    int[] orderMapping, long fileId, int startIndex, int length)
     {
         this.id = id;
         this.schema = schema;
-        this.rowBatch = schema.createRowBatchWithHiddenColumn(size);
+        this.orderMapping = orderMapping;
+        this.rowBatch = schema.createRowBatchWithHiddenColumn(size, vectorLayout);
         this.fileId = fileId;
         this.startIndex = startIndex;
         this.length = length;
@@ -68,13 +70,14 @@ public synchronized int add(byte[][] values, long timestamp) throws RetinaExcept
         }
         for (int i = 0; i < values.length; ++i)
         {
-            if (values[i] == null)
+            byte[] value = values[this.orderMapping[i]];
+            if (value == null)
             {
                 this.rowBatch.cols[i].addNull();
             }
             else
             {
-                this.rowBatch.cols[i].add(values[i]);
+                this.rowBatch.cols[i].add(value);
             }
         }
         this.rowBatch.cols[schema.getChildren().size()].add(timestamp);

pixels-retina/src/main/java/io/pixelsdb/pixels/retina/PixelsWriteBuffer.java

@@ -69,6 +69,9 @@ public class PixelsWriteBuffer
     // column information is recorded to create rowBatch
     private final TypeDescription schema;
 
+    // column order mapping from layout to metadata column order
+    private final int[] orderMapping;
+
     // configuration information of PixelsWriter
     private final int memTableSize;
     private final long blockSize;
@@ -130,11 +133,13 @@ public class PixelsWriteBuffer
     private final int virtualNodeId;
     private final IndexOption indexOption;
 
-    public PixelsWriteBuffer(long tableId, TypeDescription schema, Path targetOrderedDirPath,
-                             Path targetCompactDirPath, String retinaHostName, int virtualNode) throws RetinaException
+    public PixelsWriteBuffer(long tableId, TypeDescription schema, int[] orderMapping,
+                             Path targetOrderedDirPath, Path targetCompactDirPath,
+                             String retinaHostName, int virtualNode) throws RetinaException
     {
         this.tableId = tableId;
         this.schema = schema;
+        this.orderMapping = orderMapping;
         this.virtualNodeId = virtualNode;
         this.indexOption = IndexOption.builder()
                 .vNodeId(virtualNodeId)
@@ -181,8 +186,9 @@ public PixelsWriteBuffer(long tableId, TypeDescription schema, Path targetOrdere
                 idCounter, this.memTableSize * this.maxMemTableCount, retinaHostName, virtualNodeId);
         this.ingestFilePublisher = new IngestFilePublisher(this.currentFileWriterManager.getFirstBlockId());
 
-        this.activeMemTable = new MemTable(this.idCounter, schema, memTableSize, this.currentFileWriterManager.getFileId(),
-                0, this.memTableSize);
+        this.activeMemTable = new MemTable(this.idCounter, schema, memTableSize,
+                TypeDescription.VectorLayout.NONE, this.orderMapping,
+                this.currentFileWriterManager.getFileId(), 0, this.memTableSize);
         this.idCounter++;
         this.currentMemTableCount = 1;
 
@@ -297,7 +303,9 @@ private void retireActiveMemTableLocked() throws RetinaException
         MemTable oldMemTable = this.activeMemTable;
         SuperVersion oldVersion = this.currentVersion;
         this.immutableMemTables.add(this.activeMemTable);
-        this.activeMemTable = new MemTable(this.idCounter, this.schema, this.memTableSize, this.currentFileWriterManager.getFileId(),
+        this.activeMemTable = new MemTable(this.idCounter, this.schema,
+                this.memTableSize, TypeDescription.VectorLayout.NONE,
+                this.orderMapping, this.currentFileWriterManager.getFileId(),
                 this.currentMemTableCount * this.memTableSize,
                 this.memTableSize);
         this.currentMemTableCount += 1;

pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RetinaResourceManager.java

@@ -680,10 +680,28 @@ public void addWriteBuffer(String schemaName, String tableName) throws RetinaExc
             List<io.pixelsdb.pixels.common.metadata.domain.Path> orderedPaths = latestLayout.getOrderedPaths();
             List<io.pixelsdb.pixels.common.metadata.domain.Path> compactPaths = latestLayout.getCompactPaths();
 
-            // get schema
+            // Build the file schema in layout order while retaining a mapping from
+            // layout positions to the metadata order used by Retina RPC values.
             List<Column> columns = this.metadataService.getColumns(schemaName, tableName, false);
-            List<String> columnNames = columns.stream().map(Column::getName).collect(Collectors.toList());
-            List<String> columnTypes = columns.stream().map(Column::getType).collect(Collectors.toList());
+            List<String> layoutColumnOrder = latestLayout.getOrdered().getColumnOrder();
+            List<String> metadataColumnNames = new ArrayList<>(columns.size());
+            for (Column column : columns)
+            {
+                metadataColumnNames.add(column.getName());
+            }
+            int[] orderMapping = new int[layoutColumnOrder.size()];
+            for (int i = 0; i < layoutColumnOrder.size(); ++i)
+            {
+                orderMapping[i] = metadataColumnNames.indexOf(layoutColumnOrder.get(i));
+            }
+            List<String> columnNames = new ArrayList<>(columns.size());
+            List<String> columnTypes = new ArrayList<>(columns.size());
+            for (int metadataColumnIndex : orderMapping)
+            {
+                Column column = columns.get(metadataColumnIndex);
+                columnNames.add(column.getName());
+                columnTypes.add(column.getType());
+            }
             TypeDescription schema = TypeDescription.createSchemaFromStrings(columnNames, columnTypes);
 
             String writeBufferKey = RetinaUtils.buildWriteBufferKey(schemaName, tableName);
@@ -693,7 +711,7 @@ public void addWriteBuffer(String schemaName, String tableName) throws RetinaExc
             for (int i = 0; i < totalVirtualNodeNum; i++)
             {
                 PixelsWriteBuffer pixelsWriteBuffer = new PixelsWriteBuffer(latestLayout.getTableId(),
-                        schema, orderedPaths.get(0), compactPaths.get(0), retinaHostName, i);
+                        schema, orderMapping, orderedPaths.get(0), compactPaths.get(0), retinaHostName, i);
                 nodeBuffers.put(i, pixelsWriteBuffer);
             }
         } catch (Exception e)

pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestPixelsWriteBuffer.java

@@ -69,7 +69,14 @@ public void testConcurrentWriteOperations()
     {
         try
         {
-            buffer = new PixelsWriteBuffer(0L, schema, targetOrderDirPath, targetCompactDirPath, "localhost", 0);  // table id get from mysql `TBLS` table
+            int[] orderMapping = new int[schema.getChildren().size()];
+            for (int i = 0; i < orderMapping.length; ++i)
+            {
+                orderMapping[i] = i;
+            }
+            // table id get from mysql `TBLS` table
+            buffer = new PixelsWriteBuffer(0L, schema, orderMapping, targetOrderDirPath,
+                    targetCompactDirPath, "localhost", 0);
         } catch (Exception e)
         {
             System.out.println("setup error: " + e);
@@ -154,7 +161,9 @@ private static MemTable newMemTable(int size)
     {
         TypeDescription schema = TypeDescription.createSchemaFromStrings(
                 Arrays.asList("id"), Arrays.asList("int"));
-        return new MemTable(0L, schema, size, 100L, 0, size);
+        int[] orderMapping = new int[] {0};
+        return new MemTable(0L, schema, size, TypeDescription.VectorLayout.NONE,
+            orderMapping, 100L, 0, size);
     }
 
     private static byte[][] row(int value)