diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cad6358..362998d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - java-version: [11, 21] + java-version: [11, 25] steps: - uses: actions/checkout@v7 - name: Set up Java @@ -33,6 +33,8 @@ jobs: git clone --depth 1 https://github.com/bytecodealliance/endive.git /tmp/endive cd /tmp/endive mvn -B -Dquickly install + - name: Prepare wast files + run: ./update-spec-tests.sh - name: Build and test run: mvn -B install - name: Publish Test Report diff --git a/.gitignore b/.gitignore index f924c3f..bd4c723 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ target/ .DS_Store .claude +/parser/src/test/resources/spec-tests/ diff --git a/Readme.md b/Readme.md index 691e986..a658d07 100644 --- a/Readme.md +++ b/Readme.md @@ -38,6 +38,9 @@ The long-term goal is to bring full Component Model support to Endive: ## Current Status +Initial work is in progress on the component type model (`types` module), binary parser (`parser` module), and +supporting infrastructure including initial `wasm-tools component` command support (`wasm-tools` module). + The repository includes a `wit-parser` module that wraps the [wasm-tools](https://github.com/bytecodealliance/wasm-tools) `component wit` command, using the same pattern as the @@ -65,8 +68,13 @@ mvn -Dquickly Then build endive-cm: +The tests of the `parser` module rely upon downloading a local copy of the .wast tests from the Component Model spec +repository. A shell script is provided that fetches the most recent copy of the .wast tests. + +To build endive-cm: ```sh cd endive-cm +./update-spec-tests.sh mvn clean install ``` diff --git a/docs/phases/00-type-model.md b/docs/phases/00-type-model.md index 4fcad90..c47a2bb 100644 --- a/docs/phases/00-type-model.md +++ b/docs/phases/00-type-model.md @@ -1,6 +1,6 @@ # Phase 0: Type Model Foundation -**Status**: Not started +**Status**: In progress ## Goal diff --git a/docs/phases/01-binary-parser.md b/docs/phases/01-binary-parser.md index b652405..28084a3 100644 --- a/docs/phases/01-binary-parser.md +++ b/docs/phases/01-binary-parser.md @@ -1,6 +1,6 @@ # Phase 1: Binary Parser -**Status**: Not started +**Status**: In progress **Depends on**: Phase 0 (Type Model) ## Goal @@ -81,6 +81,9 @@ and immutable fields for all sections. ## Testing +- **Component Model spec tests**: sync .wast tests from the [Component Model spec tests](https://github.com/WebAssembly/component-model/tree/main/test) + and use `wasm-tools json-from-wast` to extract and execute tests, parse with `ComponentParser`, and verify the .wast + assertions - **Round-trip with wasm-tools**: produce component binaries via `wasm-tools component new`, parse with `ComponentParser`, verify structure - **Error paths**: truncated input, wrong magic, wrong layer, malformed diff --git a/docs/phases/02-wasm-tools.md b/docs/phases/02-wasm-tools.md index 4ff8232..bb1019c 100644 --- a/docs/phases/02-wasm-tools.md +++ b/docs/phases/02-wasm-tools.md @@ -1,6 +1,6 @@ # Phase 2: wasm-tools Integration (Component Commands) -**Status**: Not started +**Status**: In progress **Depends on**: None (independent, but Phase 1 testing benefits from it) ## Goal diff --git a/parser/pom.xml b/parser/pom.xml new file mode 100644 index 0000000..9d4be98 --- /dev/null +++ b/parser/pom.xml @@ -0,0 +1,88 @@ + + + 4.0.0 + + run.endive.cm + endive-cm + 999-SNAPSHOT + + parser + jar + + Endive CM - Binary Component Model Parser + Binary parser for Endive Component Model + + + + run.endive + wasm + + + run.endive.cm + types + + + com.fasterxml.jackson.core + jackson-databind + test + + + io.roastedroot + zerofs + test + + + org.assertj + assertj-core + test + + + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-params + test + + + run.endive + log + test + + + run.endive + runtime + test + + + run.endive + wasi + test + + + run.endive + wasm-tools + test + + + run.endive.cm + wasm-tools + test + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 16 + 16 + + + + + + diff --git a/parser/src/main/java/run/endive/cm/parser/ComponentParser.java b/parser/src/main/java/run/endive/cm/parser/ComponentParser.java new file mode 100644 index 0000000..8212a75 --- /dev/null +++ b/parser/src/main/java/run/endive/cm/parser/ComponentParser.java @@ -0,0 +1,325 @@ +package run.endive.cm.parser; + +import static java.util.Objects.requireNonNull; +import static run.endive.cm.parser.CoreParser.parseCustomSection; +import static run.endive.cm.parser.CoreParser.parseRecType; +import static run.endive.wasm.Encoding.readVarUInt32; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Arrays; +import java.util.function.Supplier; +import run.endive.cm.types.CoreAlias; +import run.endive.cm.types.CoreExportDecl; +import run.endive.cm.types.CoreImportDecl; +import run.endive.cm.types.CoreOuterAliasTarget; +import run.endive.cm.types.CoreSort; +import run.endive.cm.types.CoreType; +import run.endive.cm.types.CoreTypeSection; +import run.endive.cm.types.CustomSection; +import run.endive.cm.types.ModuleDecl; +import run.endive.cm.types.ModuleType; +import run.endive.cm.types.Section; +import run.endive.cm.types.SectionId; +import run.endive.cm.types.WasmComponent; +import run.endive.wasm.MalformedException; +import run.endive.wasm.io.InputStreams; + +public final class ComponentParser { + + static final byte[] MAGIC_BYTES = {0x00, 0x61, 0x73, 0x6D}; + static final byte[] VERSION_BYTES = {0x0d, 0x00}; + static final byte[] LAYER_BYTES = {0x01, 0x00}; + + private ComponentParser() {} + + private static ByteBuffer readByteBuffer(InputStream is) { + try { + var buffer = ByteBuffer.wrap(InputStreams.readAllBytes(is)); + buffer.order(ByteOrder.LITTLE_ENDIAN); + return buffer; + } catch (IOException e) { + throw new IllegalArgumentException("Failed to read wasm bytes.", e); + } + } + + public static ComponentParser.Builder builder() { + return new ComponentParser.Builder(); + } + + public static final class Builder { + + private Builder() {} + + public ComponentParser build() { + return new ComponentParser(); + } + } + + private static void onSection(WasmComponent.Builder module, Section s) { + switch (s.sectionId()) { + case SectionId.CUSTOM: + var coreCustomSection = (CustomSection) s; + module.addCoreCustomSection(coreCustomSection); + break; + case SectionId.CORE_TYPE: + var coreTypeSection = (CoreTypeSection) s; + module.addCoreTypeSection(coreTypeSection); + break; + default: + throw new MalformedException("unsupported section id " + s.sectionId()); + } + } + + public WasmComponent parse(Supplier inputStreamSupplier) { + WasmComponent.Builder componentBuilder = WasmComponent.builder(); + parse(inputStreamSupplier.get(), s -> onSection(componentBuilder, s)); + return componentBuilder.build(); + } + + private void parse(InputStream in, ComponentParserListener listener) { + requireNonNull(listener, "listener"); + + var buffer = readByteBuffer(in); + + byte[] magic = new byte[4]; + readBytes(buffer, magic); + if (!Arrays.equals(magic, MAGIC_BYTES)) { + throw new MalformedException( + "magic header not detected, found: " + + Arrays.toString(magic) + + " expected: " + + Arrays.toString(MAGIC_BYTES)); + } + + byte[] version = new byte[2]; + readBytes(buffer, version); + if (!Arrays.equals(version, VERSION_BYTES)) { + throw new MalformedException( + "unknown binary version, found: " + + Arrays.toString(version) + + " expected: " + + Arrays.toString(VERSION_BYTES)); + } + + byte[] layer = new byte[2]; + readBytes(buffer, layer); + if (!Arrays.equals(layer, LAYER_BYTES)) { + throw new MalformedException( + "unknown layer, found: " + + Arrays.toString(layer) + + " expected: " + + Arrays.toString(LAYER_BYTES)); + } + + while (buffer.hasRemaining()) { + var sectionId = readByte(buffer); + var sectionSize = readVarUInt32(buffer); + + ByteBuffer sectionByteBuffer = buffer.asReadOnlyBuffer(); + sectionByteBuffer.order(buffer.order()); + + // move buffer to next section + var sectionLimit = sectionByteBuffer.position() + (int) sectionSize; + if (buffer.capacity() < sectionLimit) { + throw new MalformedException("length out of bounds for section" + sectionId); + } + buffer.position(sectionLimit); + + sectionByteBuffer.limit(sectionLimit); + + // Process different section types based on the sectionId + switch (sectionId) { + case SectionId.CUSTOM: + { + var customSection = + parseCustomSection(sectionByteBuffer, sectionSize, true); + var coreCustomSection = + CustomSection.builder().withCustomSection(customSection).build(); + listener.onSection(coreCustomSection); + break; + } + case SectionId.CORE_MODULE: + { + throw new UnsupportedOperationException( + "Core module section is not supported yet"); + } + case SectionId.CORE_INSTANCE: + { + throw new UnsupportedOperationException( + "Core instance section is not supported yet"); + } + case SectionId.CORE_TYPE: + { + var coreTypeSection = parseCoreTypeSection(sectionByteBuffer); + listener.onSection(coreTypeSection); + break; + } + case SectionId.COMPONENT: + { + throw new UnsupportedOperationException( + "Component section is not supported yet"); + } + case SectionId.INSTANCE: + { + throw new UnsupportedOperationException( + "Instance section is not supported yet"); + } + case SectionId.ALIAS: + { + throw new UnsupportedOperationException( + "Alias section is not supported yet"); + } + case SectionId.TYPE: + { + throw new UnsupportedOperationException( + "Type section is not supported yet"); + } + case SectionId.CANON: + { + throw new UnsupportedOperationException( + "Canon section is not supported yet"); + } + case SectionId.START: + { + throw new UnsupportedOperationException( + "Start section is not supported yet"); + } + case SectionId.IMPORT: + { + throw new UnsupportedOperationException( + "Import section is not supported yet"); + } + case SectionId.EXPORT: + { + throw new UnsupportedOperationException( + "Export section is not supported yet"); + } + case SectionId.VALUE: + { + throw new UnsupportedOperationException( + "Value section is not supported yet"); + } + default: + { + throw new MalformedException( + "section size mismatch, malformed section id " + sectionId); + } + } + + if (sectionByteBuffer.hasRemaining()) { + throw new MalformedException("section size mismatch"); + } + } + } + + private static CoreTypeSection parseCoreTypeSection(ByteBuffer buffer) { + var builder = CoreTypeSection.builder(); + var numCoreTypes = readVarUInt32(buffer); + for (int i = 0; i < numCoreTypes && buffer.hasRemaining(); i++) { + builder.addCoreType(parseCoreType(buffer)); + } + return builder.build(); + } + + private static CoreType parseCoreType(ByteBuffer buffer) { + var typeBuilder = CoreType.builder(); + var opcode = peekByte(buffer); + switch (opcode) { + case 0x50: + buffer.position(buffer.position() + 1); + typeBuilder.withModuleType(parseModuleType(buffer)); + break; + case 0x00: + buffer.position(buffer.position() + 1); + typeBuilder.withRecType(parseRecType(buffer)); + break; + default: + typeBuilder.withRecType(parseRecType(buffer)); + } + return typeBuilder.build(); + } + + private static ModuleType parseModuleType(ByteBuffer buffer) { + var builder = ModuleType.builder(); + var numDecls = readVarUInt32(buffer); + for (int i = 0; i < numDecls && buffer.hasRemaining(); i++) { + builder.addModuleDecl(parseModuleDecl(buffer)); + } + return builder.build(); + } + + private static ModuleDecl parseModuleDecl(ByteBuffer buffer) { + var builder = ModuleDecl.builder(); + var opcode = readByte(buffer); + switch (opcode) { + case 0x00: + builder.withImportDecl(parseCoreImportDecl(buffer)); + break; + case 0x01: + builder.withType(parseCoreType(buffer)); + break; + case 0x02: + builder.withAlias(parseCoreAlias(buffer)); + break; + case 0x03: + builder.withExportDecl(parseCoreExportDecl(buffer)); + break; + default: + throw new MalformedException("unknown opcode" + opcode + " in module decl"); + } + return builder.build(); + } + + private static CoreImportDecl parseCoreImportDecl(ByteBuffer buffer) { + throw new UnsupportedOperationException("import decl parsing not implemented"); + } + + private static CoreAlias parseCoreAlias(ByteBuffer buffer) { + var builder = CoreAlias.builder(); + var sortId = readByte(buffer); + var sort = CoreSort.fromId(sortId); + var targetOpcode = readByte(buffer); + if (targetOpcode != 0x01) { + throw new MalformedException("unknown target opcode " + targetOpcode + " in alias"); + } + var typeIndex = readVarUInt32(buffer); + var sortIndex = readVarUInt32(buffer); + builder.withSort(sort); + builder.withOuterTarget( + CoreOuterAliasTarget.builder() + .withTypeIndex(typeIndex) + .withSortIndex(sortIndex) + .build()); + return builder.build(); + } + + private static CoreExportDecl parseCoreExportDecl(ByteBuffer buffer) { + throw new UnsupportedOperationException("export decl parsing not implemented"); + } + + static byte readByte(ByteBuffer buffer) { + if (!buffer.hasRemaining()) { + throw new MalformedException("length out of bounds"); + } + + return buffer.get(); + } + + static byte peekByte(ByteBuffer buffer) { + if (!buffer.hasRemaining()) { + throw new MalformedException("length out of bounds"); + } + + return buffer.get(buffer.position()); + } + + static void readBytes(ByteBuffer buffer, byte[] dest) { + if (buffer.remaining() < dest.length) { + throw new MalformedException("length out of bounds"); + } + buffer.get(dest); + } +} diff --git a/parser/src/main/java/run/endive/cm/parser/ComponentParserListener.java b/parser/src/main/java/run/endive/cm/parser/ComponentParserListener.java new file mode 100644 index 0000000..ee38fde --- /dev/null +++ b/parser/src/main/java/run/endive/cm/parser/ComponentParserListener.java @@ -0,0 +1,9 @@ +package run.endive.cm.parser; + +import run.endive.cm.types.Section; + +@FunctionalInterface +public interface ComponentParserListener { + + void onSection(Section section); +} diff --git a/parser/src/main/java/run/endive/cm/parser/CoreParser.java b/parser/src/main/java/run/endive/cm/parser/CoreParser.java new file mode 100644 index 0000000..80516a0 --- /dev/null +++ b/parser/src/main/java/run/endive/cm/parser/CoreParser.java @@ -0,0 +1,175 @@ +package run.endive.cm.parser; + +import static run.endive.cm.parser.ComponentParser.readByte; +import static run.endive.cm.parser.ComponentParser.readBytes; +import static run.endive.wasm.Encoding.readName; +import static run.endive.wasm.Encoding.readVarSInt32; +import static run.endive.wasm.Encoding.readVarUInt32; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import run.endive.wasm.MalformedException; +import run.endive.wasm.types.ArrayType; +import run.endive.wasm.types.CompType; +import run.endive.wasm.types.CustomSection; +import run.endive.wasm.types.FieldType; +import run.endive.wasm.types.FunctionType; +import run.endive.wasm.types.MutabilityType; +import run.endive.wasm.types.PackedType; +import run.endive.wasm.types.RecType; +import run.endive.wasm.types.StorageType; +import run.endive.wasm.types.StructType; +import run.endive.wasm.types.SubType; +import run.endive.wasm.types.UnknownCustomSection; +import run.endive.wasm.types.ValType; + +public final class CoreParser { + + private CoreParser() {} + + static CustomSection parseCustomSection( + ByteBuffer buffer, long sectionSize, boolean checkMalformed) { + var sectionPos = buffer.position(); + var name = readName(buffer, checkMalformed); + var size = (sectionSize - (buffer.position() - sectionPos)); + if (size < 0) { + throw new MalformedException("unexpected end"); + } + var bytes = new byte[(int) size]; + readBytes(buffer, bytes); + return UnknownCustomSection.builder().withName(name).withBytes(bytes).build(); + } + + static FieldType parseFieldType(ByteBuffer buffer) { + var id = (int) readVarUInt32(buffer); + + if (id == PackedType.I8.ID() || id == PackedType.I16.ID()) { + var packedType = PackedType.fromId(id); + var mut = MutabilityType.forId(readByte(buffer)); + return FieldType.builder() + .withStorageType(StorageType.builder().withPackedType(packedType).build()) + .withMutability(mut) + .build(); + } else { + var valType = readValueTypeBuilderFromOpCode(buffer, id).build(); + var mut = MutabilityType.forId(readByte(buffer)); + + return FieldType.builder() + .withStorageType(StorageType.builder().withValType(valType).build()) + .withMutability(mut) + .build(); + } + } + + static ArrayType parseArrayType(ByteBuffer buffer) { + return ArrayType.builder().withFieldType(parseFieldType(buffer)).build(); + } + + static StructType parseStructType(ByteBuffer buffer) { + var count = (int) readVarUInt32(buffer); + var builder = StructType.builder(); + for (int i = 0; i < count; i++) { + builder.addFieldType(parseFieldType(buffer)); + } + return builder.build(); + } + + static FunctionType parseFunctionType(ByteBuffer buffer) { + var paramCount = (int) readVarUInt32(buffer); + List paramsBuilder = new ArrayList<>(paramCount); + + // Parse parameter types + for (int j = 0; j < paramCount; j++) { + paramsBuilder.add(readValueTypeBuilder(buffer).build()); + } + + var returnCount = (int) readVarUInt32(buffer); + List returnsBuilder = new ArrayList<>(returnCount); + + // Parse return types + for (int j = 0; j < returnCount; j++) { + returnsBuilder.add(readValueTypeBuilder(buffer).build()); + } + + return FunctionType.of(paramsBuilder, returnsBuilder); + } + + static CompType parseCompType(int id, ByteBuffer buffer) { + if (id > Byte.MAX_VALUE) { + throw new MalformedException("integer representation too long"); + } + + switch (id) { + case 0x5E: + return CompType.builder().withArrayType(parseArrayType(buffer)).build(); + case 0x5F: + return CompType.builder().withStructType(parseStructType(buffer)).build(); + case 0x60: + return CompType.builder().withFuncType(parseFunctionType(buffer)).build(); + default: + throw new MalformedException( + "Invalid composite type. Form " + + String.format("0x%02X", id) + + " was not 0x5E, 0x5f or 0x60"); + } + } + + static SubType parseSubType(int id, ByteBuffer buffer) { + if (id == 0x50 // non final typeIdx + || id == 0x4F // final typeIdx + ) { + var count = (int) readVarUInt32(buffer); + var typeIdxs = new int[count]; + + for (int i = 0; i < count; i++) { + typeIdxs[i] = (int) readVarUInt32(buffer); + } + return SubType.builder() + .withTypeIdx(typeIdxs) + .withFinal(id == 0x4F) + .withCompType(parseCompType((int) readVarUInt32(buffer), buffer)) + .build(); + } else { + // fallback to the compressed form + return SubType.builder() + .withTypeIdx(new int[0]) + .withFinal(true) + .withCompType(parseCompType(id, buffer)) + .build(); + } + } + + static RecType parseRecType(ByteBuffer buffer) { + var discriminator = (int) readVarUInt32(buffer); + if (discriminator == 0x4E) { + var count = (int) readVarUInt32(buffer); + var subTypes = new SubType[count]; + + for (int i = 0; i < count; i++) { + subTypes[i] = parseSubType((int) readVarUInt32(buffer), buffer); + } + return RecType.builder().withSubTypes(subTypes).build(); + } else { + // fallback to the compressed form + return RecType.builder() + .withSubTypes(new SubType[] {parseSubType(discriminator, buffer)}) + .build(); + } + } + + static ValType.Builder readValueTypeBuilderFromOpCode(ByteBuffer buffer, int valueTypeOpCode) { + var builder = ValType.builder().withOpcode(valueTypeOpCode); + if (valueTypeOpCode == ValType.ID.Ref || valueTypeOpCode == ValType.ID.RefNull) { + return builder.withTypeIdx((int) readVarSInt32(buffer)); + } else { + return builder; + } + } + + static ValType.Builder readValueTypeBuilder(ByteBuffer buffer) { + var valueTypeOpCode = (int) readVarUInt32(buffer); + + return readValueTypeBuilderFromOpCode(buffer, valueTypeOpCode); + } +} diff --git a/parser/src/test/java/run/endive/cm/parser/ComponentParserTests.java b/parser/src/test/java/run/endive/cm/parser/ComponentParserTests.java new file mode 100644 index 0000000..bd4fe7b --- /dev/null +++ b/parser/src/test/java/run/endive/cm/parser/ComponentParserTests.java @@ -0,0 +1,97 @@ +package run.endive.cm.parser; + +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Map; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import run.endive.cm.types.CoreAlias; +import run.endive.cm.types.CoreOuterAliasTarget; +import run.endive.cm.types.CoreSort; +import run.endive.cm.types.CoreType; +import run.endive.cm.types.CoreTypeSection; +import run.endive.cm.types.ModuleDecl; +import run.endive.cm.types.ModuleType; +import run.endive.cm.types.WasmComponent; +import run.endive.wasm.types.CompType; +import run.endive.wasm.types.FunctionType; +import run.endive.wasm.types.RecType; +import run.endive.wasm.types.SubType; + +public class ComponentParserTests { + + static java.util.stream.Stream testLocations() { + + return java.util.stream.Stream.of( + Arguments.arguments( + "./wasm-tools/types.wast", + WasmToolsTypesAssertions.expectedComponents, + new int[] {29, 30, 31, 32, 33, 34, 40})); + } + + @ParameterizedTest + @MethodSource("testLocations") + void allSpecTests( + String testLocation, Map expectedComponents, int[] skipCases) + throws IOException { + var testDir = Path.of("src/test/resources/spec-tests"); + var testFile = testDir.resolve(testLocation); + try (var wast = Files.newInputStream(testFile); + var wastTests = JsonFromWast.exec(wast)) { + wastTests.runTests(expectedComponents, true, skipCases); + } catch (Throwable e) { + if (e instanceof WastTests.CommandException) { + var ce = (WastTests.CommandException) e; + var outFile = testDir.resolve("./failed.wasm"); + Files.write(outFile, ce.testContents()); + } + fail(e); + } + } + + private static final class WasmToolsTypesAssertions { + + private static final Map expectedComponents = Map.of(17, case17()); + + private static WasmComponent case17() { + var compType = + CompType.builder() + .withFuncType(FunctionType.of(new ArrayList<>(), new ArrayList<>())) + .build(); + var subType = + SubType.builder() + .withTypeIdx(new int[] {}) + .withFinal(true) + .withCompType(compType) + .build(); + var recType = RecType.builder().withSubTypes(new SubType[] {subType}).build(); + var coreFuncType = CoreType.builder().withRecType(recType).build(); + + var alias = + CoreAlias.builder() + .withSort(CoreSort.TYPE) + .withOuterTarget( + CoreOuterAliasTarget.builder() + .withTypeIndex(1) + .withSortIndex(0) + .build()) + .build(); + + var moduleDecl = ModuleDecl.builder().withAlias(alias).build(); + var moduleType = ModuleType.builder().addModuleDecl(moduleDecl).build(); + var coreModuleType = CoreType.builder().withModuleType(moduleType).build(); + + var coreTypeSection = + CoreTypeSection.builder() + .addCoreType(coreFuncType) + .addCoreType(coreModuleType) + .build(); + return WasmComponent.builder().addCoreTypeSection(coreTypeSection).build(); + } + } +} diff --git a/parser/src/test/java/run/endive/cm/parser/JsonFromWast.java b/parser/src/test/java/run/endive/cm/parser/JsonFromWast.java new file mode 100644 index 0000000..9809fd9 --- /dev/null +++ b/parser/src/test/java/run/endive/cm/parser/JsonFromWast.java @@ -0,0 +1,95 @@ +package run.endive.cm.parser; + +import io.roastedroot.zerofs.Configuration; +import io.roastedroot.zerofs.ZeroFs; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import run.endive.cm.tools.ComponentValidateException; +import run.endive.log.Logger; +import run.endive.log.SystemLogger; +import run.endive.runtime.ByteArrayMemory; +import run.endive.runtime.ImportValues; +import run.endive.runtime.Instance; +import run.endive.tools.wasm.WasmToolsModule; +import run.endive.wasi.WasiExitException; +import run.endive.wasi.WasiOptions; +import run.endive.wasi.WasiPreview1; +import run.endive.wasm.WasmModule; + +public final class JsonFromWast { + + private JsonFromWast() {} + + private static final Logger logger = + new SystemLogger() { + @Override + public boolean isLoggable(Logger.Level level) { + return false; + } + }; + + private static final WasmModule MODULE = WasmToolsModule.load(); + + public static WastTests exec(InputStream is) { + try (var stdinStream = new ByteArrayInputStream(new byte[0]); + var stdoutStream = new ByteArrayOutputStream(); + var stderrStream = new ByteArrayOutputStream()) { + + FileSystem fs = + ZeroFs.newFileSystem( + Configuration.unix().toBuilder().setAttributeViews("unix").build()); + + Path wastDir = fs.getPath("wast"); + Files.createDirectory(wastDir); + Path inputFile = wastDir.resolve("input.wast"); + byte[] source = is.readAllBytes(); + Files.write(inputFile, source); + + var options = + WasiOptions.builder() + .withStdin(stdinStream, false) + .withStdout(stdoutStream, false) + .withStderr(stderrStream, false) + .withDirectory("/", fs.getPath("/work")) + .withArguments( + List.of( + "wasm-tools", + "json-from-wast", + inputFile.toString(), + "-vv", + "--pretty")) + .build(); + + try (var wasi = + WasiPreview1.builder().withLogger(logger).withOptions(options).build()) { + var imports = ImportValues.builder().addFunction(wasi.toHostFunctions()).build(); + try { + Instance.builder(MODULE) + .withMachineFactory(WasmToolsModule::create) + .withMemoryFactory(ByteArrayMemory::new) + .withImportValues(imports) + .build(); + } catch (WasiExitException e) { + if (e.exitCode() != 0) { + throw new ComponentValidateException( + stdoutStream.toString(StandardCharsets.UTF_8) + + stderrStream.toString(StandardCharsets.UTF_8), + e); + } + } + var wastJson = stdoutStream.toString(StandardCharsets.UTF_8); + return new WastTests(wastJson, fs); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/parser/src/test/java/run/endive/cm/parser/JsonFromWastException.java b/parser/src/test/java/run/endive/cm/parser/JsonFromWastException.java new file mode 100644 index 0000000..2ef04ac --- /dev/null +++ b/parser/src/test/java/run/endive/cm/parser/JsonFromWastException.java @@ -0,0 +1,14 @@ +package run.endive.cm.parser; + +public class JsonFromWastException extends RuntimeException { + + public JsonFromWastException() {} + + public JsonFromWastException(String message) { + super(message); + } + + public JsonFromWastException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/parser/src/test/java/run/endive/cm/parser/WastTests.java b/parser/src/test/java/run/endive/cm/parser/WastTests.java new file mode 100644 index 0000000..519651d --- /dev/null +++ b/parser/src/test/java/run/endive/cm/parser/WastTests.java @@ -0,0 +1,413 @@ +package run.endive.cm.parser; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.math.BigInteger; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.stream.IntStream; +import run.endive.cm.tools.ComponentValidate; +import run.endive.cm.tools.ComponentValidateException; +import run.endive.cm.types.WasmComponent; + +public final class WastTests implements AutoCloseable { + + private final WastTestFile wastTestFile; + + private final FileSystem wastFS; + + public WastTests(String wastJson, FileSystem wastFS) { + var objectMapper = new ObjectMapper(); + try { + this.wastTestFile = objectMapper.readValue(wastJson, WastTestFile.class); + this.wastFS = wastFS; + } catch (JsonProcessingException e) { + throw new JsonFromWastException("failed to deserialize wast json", e); + } + } + + public void runTests( + Map expectedComponents, boolean parseOnly, int[] skipCases) { + var workingDir = wastFS.getPath("/work"); + for (var command : wastTestFile.commands()) { + command.execute(workingDir, expectedComponents, parseOnly, skipCases); + } + } + + @Override + public void close() throws Exception { + this.wastFS.close(); + } + + static final class WastTestFile { + private final String sourceFilename; + private final List commands; + + WastTestFile( + @JsonProperty("source_filename") String sourceFilename, + @JsonProperty("commands") List commands) { + this.sourceFilename = sourceFilename; + this.commands = commands; + } + + String sourceFilename() { + return sourceFilename; + } + + List commands() { + return commands; + } + } + + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") + @JsonSubTypes({ + @JsonSubTypes.Type(value = Module.class, name = "module"), + @JsonSubTypes.Type(value = AssertMalformed.class, name = "assert_malformed"), + @JsonSubTypes.Type(value = AssertInvalid.class, name = "assert_invalid") + }) + interface Command { + + void execute( + Path location, + Map expectedComponents, + boolean parseOnly, + int... skipCases) + throws CommandException; + + default WasmComponent parseComponent(Path location, String filename) + throws CommandException { + var filePath = location.resolve(filename); + try { + byte[] bytes = Files.readAllBytes(filePath); + ComponentValidate.validate(new ByteArrayInputStream(bytes)); + var parser = ComponentParser.builder().build(); + return parser.parse(() -> new ByteArrayInputStream(bytes)); + } catch (UnsupportedOperationException e) { + throw new CommandException(filePath, e); + } catch (IOException e) { + throw new CommandException(filePath, e); + } + } + } + + public static class CommandException extends RuntimeException { + + private final byte[] testContents; + + public CommandException(Path filePath, Throwable cause) { + super(cause); + this.testContents = readFileContents(filePath); + } + + public CommandException(Path filePath, String message) { + super(message); + this.testContents = readFileContents(filePath); + } + + public CommandException(Path filePath, String message, Throwable cause) { + super(message, cause); + this.testContents = readFileContents(filePath); + } + + private byte[] readFileContents(Path location) { + try { + return Files.readAllBytes(location); + } catch (IOException e) { + throw new IllegalStateException("Failed to read test contents from " + location, e); + } + } + + public byte[] testContents() { + return testContents; + } + } + + static final class AssertInvalid implements Command { + private final int line; + private final String filename; + private final String moduleType; + private final String text; + + AssertInvalid( + @JsonProperty("line") int line, + @JsonProperty("filename") String filename, + @JsonProperty("module_type") String moduleType, + @JsonProperty("text") String text) { + this.line = line; + this.filename = filename; + this.moduleType = moduleType; + this.text = text; + } + + int line() { + return line; + } + + String filename() { + return filename; + } + + String moduleType() { + return moduleType; + } + + String text() { + return text; + } + + @Override + public void execute( + Path location, + Map expectedComponents, + boolean parseOnly, + int... skipCases) + throws CommandException { + var testId = Integer.parseInt(filename.split("\\.")[1]); + if (skipCases.length > 0) { + if (IntStream.of(skipCases).anyMatch(i -> i == testId)) { + return; + } + } + try { + parseComponent(location, filename); + } catch (Throwable e) { + if (!(e instanceof ComponentValidateException)) { + throw new CommandException( + location.resolve(filename), + String.format( + "Expected validation of %s to fail at line %d due to '%s' but" + + " got unexpected exception of type %s", + filename, line, text, e.getClass().getSimpleName()), + e); + } + return; + } + throw new CommandException( + location.resolve(filename), + String.format( + "\"Expected validation of %s to fail at line %d due to '%s' ", + filename, line, text)); + } + } + + static final class AssertMalformed implements Command { + private final int line; + private final String filename; + private final String moduleType; + private final String text; + + AssertMalformed( + @JsonProperty("line") int line, + @JsonProperty("filename") String filename, + @JsonProperty("module_type") String moduleType, + @JsonProperty("text") String text) { + this.line = line; + this.filename = filename; + this.moduleType = moduleType; + this.text = text; + } + + int line() { + return line; + } + + String filename() { + return filename; + } + + String moduleType() { + return moduleType; + } + + String text() { + return text; + } + + @Override + public void execute( + Path location, + Map expectedComponents, + boolean parseOnly, + int... skipCases) + throws CommandException { + var testId = Integer.parseInt(filename.split("\\.")[1]); + if (skipCases.length > 0) { + if (IntStream.of(skipCases).anyMatch(i -> i == testId)) { + return; + } + } + try { + parseComponent(location, filename); + } catch (Throwable e) { + if (!(e instanceof ComponentValidateException)) { + throw new CommandException( + location.resolve(filename), + String.format( + "Expected validation of %s to fail at line %d due to '%s' but" + + " got unexpected exception of type %s", + filename, line, text, e.getClass().getSimpleName()), + e); + } + return; + } + throw new CommandException( + location.resolve(filename), + String.format( + "\"Expected validation of %s to fail at line %d due to '%s' ", + filename, line, text)); + } + } + + static final class Action { + private final String type; + private final String field; + private final List args; + + Action( + @JsonProperty("type") String type, + @JsonProperty("field") String field, + @JsonProperty("args") List args) { + this.type = type; + this.field = field; + this.args = args; + } + + String type() { + return type; + } + + String field() { + return field; + } + + List args() { + return args; + } + } + + static final class TypedValue { + private final String type; + private final String value; + + TypedValue(@JsonProperty("type") String type, @JsonProperty("value") String value) { + this.type = type; + this.value = value; + } + + String type() { + return type; + } + + String value() { + return value; + } + + Object converted() { + switch (type) { + case "bool": + return Boolean.parseBoolean(value); + case "s8": + return Byte.parseByte(value); + case "u8": + case "s16": + return Short.parseShort(value); + case "u16": + case "s32": + case "i32": + return Integer.parseInt(value); + case "u32": + case "s64": + case "i64": + return Long.parseLong(value); + case "u64": + return BigInteger.valueOf(Long.parseLong(value)); + case "f32": + return Float.parseFloat(value); + case "f64": + return Double.parseDouble(value); + default: + return value; + } + } + } + + static final class Module implements Command { + private final int line; + private final String name; + private final String filename; + private final String moduleType; + + Module( + @JsonProperty("line") int line, + @JsonProperty("name") String name, + @JsonProperty("filename") String filename, + @JsonProperty("module_type") String moduleType) { + this.line = line; + this.name = name; + this.filename = filename; + this.moduleType = moduleType; + } + + int line() { + return line; + } + + String name() { + return name; + } + + String filename() { + return filename; + } + + String moduleType() { + return moduleType; + } + + @Override + public void execute( + Path location, + Map expectedComponents, + boolean parseOnly, + int... skipCases) + throws CommandException { + var testId = Integer.parseInt(filename.split("\\.")[1]); + if (skipCases.length > 0) { + if (IntStream.of(skipCases).anyMatch(i -> i == testId)) { + return; + } + } + try { + WasmComponent actualComponent = parseComponent(location, filename); + if (expectedComponents.containsKey(testId)) { + WasmComponent expectedComponent = expectedComponents.get(testId); + assertThat(actualComponent) + .usingRecursiveComparison() + .ignoringFields("customSections") + .isEqualTo(expectedComponent); + } + if (!parseOnly) { + // TODO Include component instantiation here once implemented + throw new UnsupportedOperationException( + "Component instantiation not yet implemented"); + } + } catch (Throwable e) { + throw new CommandException( + location.resolve(filename), + String.format( + "Failed to load module %s due to error at line %d", filename, line), + e); + } + } + } +} diff --git a/pom.xml b/pom.xml index 689ca9a..3620b06 100644 --- a/pom.xml +++ b/pom.xml @@ -18,6 +18,9 @@ + parser + types + wasm-tools wit-parser @@ -36,6 +39,8 @@ 999-SNAPSHOT 0.1.0 + 2.18.3 + 3.27.7 5.14.4 @@ -71,12 +76,45 @@ wasm-tools ${endive.version} + + run.endive.cm + parser + ${project.version} + + + run.endive.cm + types + ${project.version} + + + run.endive.cm + wasm-tools + ${project.version} + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + test + + + org.assertj + assertj-core + ${assertj.version} + test + org.junit.jupiter junit-jupiter-api ${junit.version} test + + org.junit.jupiter + junit-jupiter-params + ${junit.version} + test + @@ -200,6 +238,17 @@ + + + + + + java21 + + [21,) + + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/types/pom.xml b/types/pom.xml new file mode 100644 index 0000000..9fd807f --- /dev/null +++ b/types/pom.xml @@ -0,0 +1,32 @@ + + + 4.0.0 + + run.endive.cm + endive-cm + 999-SNAPSHOT + + + types + jar + + Endive CM - Component Model Types + Types for Endive Component Model + + + + run.endive + wasm + + + run.endive + wasm-tools + + + org.junit.jupiter + junit-jupiter-api + test + + + + diff --git a/types/src/main/java/run/endive/cm/types/CoreAlias.java b/types/src/main/java/run/endive/cm/types/CoreAlias.java new file mode 100644 index 0000000..b6c2b96 --- /dev/null +++ b/types/src/main/java/run/endive/cm/types/CoreAlias.java @@ -0,0 +1,50 @@ +package run.endive.cm.types; + +import java.util.Objects; + +public final class CoreAlias { + + private final CoreSort sort; + private final CoreOuterAliasTarget target; + + private CoreAlias(CoreSort sort, CoreOuterAliasTarget target) { + Objects.requireNonNull(sort, "coreSort"); + Objects.requireNonNull(target, "target"); + this.sort = sort; + this.target = target; + } + + public CoreSort sort() { + return sort; + } + + public CoreOuterAliasTarget target() { + return target; + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + + private CoreSort sort; + private CoreOuterAliasTarget target; + + private Builder() {} + + public Builder withSort(CoreSort sort) { + this.sort = sort; + return this; + } + + public Builder withOuterTarget(CoreOuterAliasTarget target) { + this.target = target; + return this; + } + + public CoreAlias build() { + return new CoreAlias(sort, target); + } + } +} diff --git a/types/src/main/java/run/endive/cm/types/CoreExportDecl.java b/types/src/main/java/run/endive/cm/types/CoreExportDecl.java new file mode 100644 index 0000000..a1c7416 --- /dev/null +++ b/types/src/main/java/run/endive/cm/types/CoreExportDecl.java @@ -0,0 +1,3 @@ +package run.endive.cm.types; + +public final class CoreExportDecl {} diff --git a/types/src/main/java/run/endive/cm/types/CoreImportDecl.java b/types/src/main/java/run/endive/cm/types/CoreImportDecl.java new file mode 100644 index 0000000..fb4a58f --- /dev/null +++ b/types/src/main/java/run/endive/cm/types/CoreImportDecl.java @@ -0,0 +1,3 @@ +package run.endive.cm.types; + +public final class CoreImportDecl {} diff --git a/types/src/main/java/run/endive/cm/types/CoreOuterAliasTarget.java b/types/src/main/java/run/endive/cm/types/CoreOuterAliasTarget.java new file mode 100644 index 0000000..03b2ae7 --- /dev/null +++ b/types/src/main/java/run/endive/cm/types/CoreOuterAliasTarget.java @@ -0,0 +1,46 @@ +package run.endive.cm.types; + +public final class CoreOuterAliasTarget { + + private final long typeIndex; + private final long sortIndex; + + private CoreOuterAliasTarget(long typeIndex, long sortIndex) { + this.typeIndex = typeIndex; + this.sortIndex = sortIndex; + } + + public static Builder builder() { + return new Builder(); + } + + public long typeIndex() { + return typeIndex; + } + + public long sortIndex() { + return sortIndex; + } + + public static final class Builder { + + private long typeIndex; + private long sortIndex; + + private Builder() {} + + public Builder withTypeIndex(long typeIndex) { + this.typeIndex = typeIndex; + return this; + } + + public Builder withSortIndex(long sortIndex) { + this.sortIndex = sortIndex; + return this; + } + + public CoreOuterAliasTarget build() { + return new CoreOuterAliasTarget(typeIndex, sortIndex); + } + } +} diff --git a/types/src/main/java/run/endive/cm/types/CoreSort.java b/types/src/main/java/run/endive/cm/types/CoreSort.java new file mode 100644 index 0000000..d606ceb --- /dev/null +++ b/types/src/main/java/run/endive/cm/types/CoreSort.java @@ -0,0 +1,31 @@ +package run.endive.cm.types; + +import java.util.Arrays; + +public enum CoreSort { + FUNC(0x00), + TABLE(0x01), + MEMORY(0x02), + GLOBAL(0x03), + TAG(0x04), + TYPE(0x10), + MODULE(0x11), + INSTANCE(0x12); + + private final int id; + + CoreSort(int id) { + this.id = id; + } + + public int getId() { + return id; + } + + public static CoreSort fromId(int id) { + return Arrays.stream(CoreSort.values()) + .filter(sort -> sort.id == id) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("Unknown ID: " + id)); + } +} diff --git a/types/src/main/java/run/endive/cm/types/CoreType.java b/types/src/main/java/run/endive/cm/types/CoreType.java new file mode 100644 index 0000000..21b2aa9 --- /dev/null +++ b/types/src/main/java/run/endive/cm/types/CoreType.java @@ -0,0 +1,113 @@ +package run.endive.cm.types; + +import java.util.Objects; +import run.endive.wasm.types.CompType; +import run.endive.wasm.types.RecType; +import run.endive.wasm.types.SubType; + +public final class CoreType { + private final RecType recType; + private final SubType subType; + private final CompType compType; + private final ModuleType moduleType; + + private CoreType(RecType recType, SubType subType, CompType compType, ModuleType moduleType) { + requireExactlyOneNonNull(recType, subType, compType, moduleType); + this.recType = recType; + this.subType = subType; + this.compType = compType; + this.moduleType = moduleType; + } + + private static void requireExactlyOneNonNull(Object a, Object b, Object c, Object d) { + if ((a == null ? 0 : 1) + (b == null ? 0 : 1) + (c == null ? 0 : 1) + (d == null ? 0 : 1) + != 1) { + throw new IllegalArgumentException("Exactly one field must be filled"); + } + } + + public RecType recType() { + return recType; + } + + public SubType subType() { + return subType; + } + + public CompType compType() { + return compType; + } + + public ModuleType moduleType() { + return moduleType; + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + + private RecType recType; + private SubType subType; + private CompType compType; + private ModuleType moduleType; + + private Builder() {} + + public Builder withRecType(RecType recType) { + this.recType = recType; + return this; + } + + public Builder withSubType(SubType subType) { + this.subType = subType; + return this; + } + + public Builder withCompType(CompType compType) { + this.compType = compType; + return this; + } + + public Builder withModuleType(ModuleType moduleType) { + this.moduleType = moduleType; + return this; + } + + public CoreType build() { + return new CoreType(recType, subType, compType, moduleType); + } + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof CoreType)) { + return false; + } + CoreType coreType = (CoreType) o; + return Objects.equals(recType, coreType.recType) + && Objects.equals(subType, coreType.subType) + && Objects.equals(compType, coreType.compType) + && Objects.equals(moduleType, coreType.moduleType); + } + + @Override + public int hashCode() { + return Objects.hash(recType, subType, compType, moduleType); + } + + @Override + public String toString() { + return "CoreType{" + + "recType=" + + recType + + ", subType=" + + subType + + ", compType=" + + compType + + ", moduleType=" + + moduleType + + '}'; + } +} diff --git a/types/src/main/java/run/endive/cm/types/CoreTypeSection.java b/types/src/main/java/run/endive/cm/types/CoreTypeSection.java new file mode 100644 index 0000000..f03cc57 --- /dev/null +++ b/types/src/main/java/run/endive/cm/types/CoreTypeSection.java @@ -0,0 +1,62 @@ +package run.endive.cm.types; + +import static java.util.Objects.requireNonNull; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +public final class CoreTypeSection extends Section { + private final List coreTypes; + + private CoreTypeSection(List coreTypes) { + super(SectionId.CORE_TYPE); + this.coreTypes = List.copyOf(coreTypes); + } + + public int numCoreTypes() { + return coreTypes.size(); + } + + public List coreTypes() { + return coreTypes; + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + private final List coreTypes = new ArrayList<>(); + + private Builder() {} + + public Builder addCoreType(CoreType coreType) { + coreTypes.add(requireNonNull(coreType, "coreType")); + return this; + } + + public CoreTypeSection build() { + return new CoreTypeSection(coreTypes); + } + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof CoreTypeSection)) { + return false; + } + CoreTypeSection that = (CoreTypeSection) o; + return Objects.equals(coreTypes, that.coreTypes); + } + + @Override + public int hashCode() { + return Objects.hashCode(coreTypes); + } + + @Override + public String toString() { + return "CoreTypeSection{" + "coreTypes=" + coreTypes + '}'; + } +} diff --git a/types/src/main/java/run/endive/cm/types/CustomSection.java b/types/src/main/java/run/endive/cm/types/CustomSection.java new file mode 100644 index 0000000..be7830b --- /dev/null +++ b/types/src/main/java/run/endive/cm/types/CustomSection.java @@ -0,0 +1,36 @@ +package run.endive.cm.types; + +import static java.util.Objects.requireNonNull; + +public final class CustomSection extends Section { + + private final run.endive.wasm.types.CustomSection customSection; + + private CustomSection(run.endive.wasm.types.CustomSection customSection) { + super(SectionId.CUSTOM); + this.customSection = customSection; + } + + public run.endive.wasm.types.CustomSection customSection() { + return customSection; + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + private run.endive.wasm.types.CustomSection customSection; + + private Builder() {} + + public CustomSection build() { + return new CustomSection(customSection); + } + + public Builder withCustomSection(run.endive.wasm.types.CustomSection customSection) { + this.customSection = requireNonNull(customSection, "customSection"); + return this; + } + } +} diff --git a/types/src/main/java/run/endive/cm/types/ModuleDecl.java b/types/src/main/java/run/endive/cm/types/ModuleDecl.java new file mode 100644 index 0000000..c669e63 --- /dev/null +++ b/types/src/main/java/run/endive/cm/types/ModuleDecl.java @@ -0,0 +1,117 @@ +package run.endive.cm.types; + +import java.util.Objects; + +public final class ModuleDecl { + + private final CoreImportDecl importDecl; + + private final CoreAlias alias; + + private final CoreType type; + + private final CoreExportDecl exportDecl; + + private static void requireExactlyOneNonNull(Object a, Object b, Object c, Object d) { + if ((a == null ? 0 : 1) + (b == null ? 0 : 1) + (c == null ? 0 : 1) + (d == null ? 0 : 1) + != 1) { + throw new IllegalArgumentException("Exactly one field must be filled"); + } + } + + private ModuleDecl( + CoreImportDecl importDecl, CoreAlias alias, CoreType type, CoreExportDecl exportDecl) { + requireExactlyOneNonNull(importDecl, alias, type, exportDecl); + this.importDecl = importDecl; + this.alias = alias; + this.type = type; + this.exportDecl = exportDecl; + } + + public CoreImportDecl importDecl() { + return importDecl; + } + + public CoreAlias alias() { + return alias; + } + + public CoreType type() { + return type; + } + + public CoreExportDecl exportDecl() { + return exportDecl; + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + private CoreImportDecl importDecl; + + private CoreAlias alias; + + private CoreType type; + + private CoreExportDecl exportDecl; + + public Builder() {} + + public Builder withImportDecl(CoreImportDecl importDecl) { + this.importDecl = importDecl; + return this; + } + + public Builder withAlias(CoreAlias alias) { + this.alias = alias; + return this; + } + + public Builder withType(CoreType type) { + this.type = type; + return this; + } + + public Builder withExportDecl(CoreExportDecl exportDecl) { + this.exportDecl = exportDecl; + return this; + } + + public ModuleDecl build() { + return new ModuleDecl(importDecl, alias, type, exportDecl); + } + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof ModuleDecl)) { + return false; + } + ModuleDecl that = (ModuleDecl) o; + return Objects.equals(importDecl, that.importDecl) + && Objects.equals(alias, that.alias) + && Objects.equals(type, that.type) + && Objects.equals(exportDecl, that.exportDecl); + } + + @Override + public int hashCode() { + return Objects.hash(importDecl, alias, type, exportDecl); + } + + @Override + public String toString() { + return "ModuleDecl{" + + "importDecl=" + + importDecl + + ", alias=" + + alias + + ", type=" + + type + + ", exportDecl=" + + exportDecl + + '}'; + } +} diff --git a/types/src/main/java/run/endive/cm/types/ModuleType.java b/types/src/main/java/run/endive/cm/types/ModuleType.java new file mode 100644 index 0000000..056f5ec --- /dev/null +++ b/types/src/main/java/run/endive/cm/types/ModuleType.java @@ -0,0 +1,54 @@ +package run.endive.cm.types; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +public final class ModuleType { + + private final List moduleDecls; + + private ModuleType(List moduleDecls) { + this.moduleDecls = List.copyOf(moduleDecls); + } + + public List getModuleDecls() { + return moduleDecls; + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + private final List moduleDecls = new ArrayList<>(); + + public Builder addModuleDecl(ModuleDecl moduleDecl) { + moduleDecls.add(moduleDecl); + return this; + } + + public ModuleType build() { + return new ModuleType(moduleDecls); + } + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof ModuleType)) { + return false; + } + ModuleType that = (ModuleType) o; + return Objects.equals(moduleDecls, that.moduleDecls); + } + + @Override + public int hashCode() { + return Objects.hashCode(moduleDecls); + } + + @Override + public String toString() { + return "ModuleType{" + "moduleDecls=" + moduleDecls + '}'; + } +} diff --git a/types/src/main/java/run/endive/cm/types/Section.java b/types/src/main/java/run/endive/cm/types/Section.java new file mode 100644 index 0000000..ed3006b --- /dev/null +++ b/types/src/main/java/run/endive/cm/types/Section.java @@ -0,0 +1,13 @@ +package run.endive.cm.types; + +public abstract class Section { + private final int id; + + Section(long id) { + this.id = (int) id; + } + + public int sectionId() { + return id; + } +} diff --git a/types/src/main/java/run/endive/cm/types/SectionId.java b/types/src/main/java/run/endive/cm/types/SectionId.java new file mode 100644 index 0000000..e328d64 --- /dev/null +++ b/types/src/main/java/run/endive/cm/types/SectionId.java @@ -0,0 +1,19 @@ +package run.endive.cm.types; + +public final class SectionId { + public static final int CUSTOM = 0; + public static final int CORE_MODULE = 1; + public static final int CORE_INSTANCE = 2; + public static final int CORE_TYPE = 3; + public static final int COMPONENT = 4; + public static final int INSTANCE = 5; + public static final int ALIAS = 6; + public static final int TYPE = 7; + public static final int CANON = 8; + public static final int START = 9; + public static final int IMPORT = 10; + public static final int EXPORT = 11; + public static final int VALUE = 12; + + private SectionId() {} +} diff --git a/types/src/main/java/run/endive/cm/types/WasmComponent.java b/types/src/main/java/run/endive/cm/types/WasmComponent.java new file mode 100644 index 0000000..b529cac --- /dev/null +++ b/types/src/main/java/run/endive/cm/types/WasmComponent.java @@ -0,0 +1,72 @@ +package run.endive.cm.types; + +import static java.util.Objects.requireNonNull; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +public final class WasmComponent { + + private final List customSections; + private final List coreTypeSections; + + private WasmComponent( + List customSections, List coreTypeSections) { + this.customSections = List.copyOf(customSections); + this.coreTypeSections = List.copyOf(coreTypeSections); + } + + public static WasmComponent.Builder builder() { + return new WasmComponent.Builder(); + } + + public List coreCustomSections() { + return customSections; + } + + public List coreTypeSections() { + return coreTypeSections; + } + + public static final class Builder { + + private final List customSections = new ArrayList<>(); + private final List coreTypeSections = new ArrayList<>(); + + private Builder() {} + + public Builder addCoreCustomSection(CustomSection customSection) { + customSections.add(requireNonNull(customSection, "coreCustomSection")); + return this; + } + + public Builder addCoreTypeSection(CoreTypeSection coreTypeSection) { + coreTypeSections.add(requireNonNull(coreTypeSection, "coreTypeSection")); + return this; + } + + public WasmComponent build() { + return new WasmComponent(customSections, coreTypeSections); + } + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof WasmComponent)) { + return false; + } + WasmComponent that = (WasmComponent) o; + return Objects.equals(coreTypeSections, that.coreTypeSections); + } + + @Override + public int hashCode() { + return Objects.hashCode(coreTypeSections); + } + + @Override + public String toString() { + return "WasmComponent{" + "coreTypeSections=" + coreTypeSections + '}'; + } +} diff --git a/update-spec-tests.sh b/update-spec-tests.sh new file mode 100755 index 0000000..80da18a --- /dev/null +++ b/update-spec-tests.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO="WebAssembly/component-model" +BRANCH="main" +REMOTE_TEST_DIR="test" +LOCAL_TEST_DIR="parser/src/test/resources/spec-tests" + +API_BASE="https://api.github.com/repos/${REPO}/contents" +RAW_BASE="https://raw.githubusercontent.com/${REPO}/${BRANCH}" + +# Tracks every remote .wast path (relative to LOCAL_TEST_DIR) seen during the run +REMOTE_FILES=() + +collect_and_download() { + local api_path="$1" + local local_path="$2" + + local entries + entries=$(curl -fsSL "${API_BASE}/${api_path}") + + while IFS= read -r entry; do + local type name + type=$(printf '%s' "$entry" | grep -o '"type":"[^"]*"' | head -1 | cut -d'"' -f4) + name=$(printf '%s' "$entry" | grep -o '"name":"[^"]*"' | head -1 | cut -d'"' -f4) + + if [[ "$type" == "dir" ]]; then + collect_and_download "${api_path}/${name}" "${local_path}/${name}" + elif [[ "$type" == "file" && "$name" == *.wast ]]; then + mkdir -p "$local_path" + local dest="${local_path}/${name}" + REMOTE_FILES+=("$dest") + echo " Downloading ${api_path}/${name}" + curl -fsSL "${RAW_BASE}/${api_path}/${name}" -o "$dest" + fi + done < <(printf '%s' "$entries" | python3 -c " +import sys, json +data = json.load(sys.stdin) +for item in data: + print('{\"type\":\"' + item['type'] + '\",\"name\":\"' + item['name'] + '\"}')") +} + +mkdir -p "$LOCAL_TEST_DIR" + +echo "Fetching .wast tests from ${REPO}/${REMOTE_TEST_DIR} (branch: ${BRANCH})..." +collect_and_download "$REMOTE_TEST_DIR" "$LOCAL_TEST_DIR" + +echo "" +echo "Removing local files no longer present in remote..." +while IFS= read -r local_file; do + if [[ ! " ${REMOTE_FILES[*]} " == *" ${local_file} "* ]]; then + echo " Deleting ${local_file}" + rm -f "$local_file" + fi +done < <(find "$LOCAL_TEST_DIR" -name "*.wast" | sort) + +# Remove any directories that are now empty +find "$LOCAL_TEST_DIR" -type d -empty -delete 2>/dev/null || true + +echo "" +echo "Done. Current files:" +find "$LOCAL_TEST_DIR" -name "*.wast" | sort | sed 's|^| |' diff --git a/wasm-tools/pom.xml b/wasm-tools/pom.xml new file mode 100644 index 0000000..455c205 --- /dev/null +++ b/wasm-tools/pom.xml @@ -0,0 +1,46 @@ + + + 4.0.0 + + run.endive.cm + endive-cm + 999-SNAPSHOT + + wasm-tools + jar + + Endive CM - wasm-tools Component Commands + Wrappers of wasm-tools component commands for Endive Component Model + + + + io.roastedroot + zerofs + + + run.endive + log + + + run.endive + runtime + + + run.endive + wasi + + + run.endive + wasm + + + run.endive + wasm-tools + + + org.junit.jupiter + junit-jupiter-api + test + + + diff --git a/wasm-tools/src/main/java/run/endive/cm/tools/ComponentValidate.java b/wasm-tools/src/main/java/run/endive/cm/tools/ComponentValidate.java new file mode 100644 index 0000000..c24858a --- /dev/null +++ b/wasm-tools/src/main/java/run/endive/cm/tools/ComponentValidate.java @@ -0,0 +1,87 @@ +package run.endive.cm.tools; + +import io.roastedroot.zerofs.Configuration; +import io.roastedroot.zerofs.ZeroFs; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import run.endive.log.Logger; +import run.endive.log.SystemLogger; +import run.endive.runtime.ByteArrayMemory; +import run.endive.runtime.ImportValues; +import run.endive.runtime.Instance; +import run.endive.tools.wasm.WasmToolsModule; +import run.endive.wasi.WasiExitException; +import run.endive.wasi.WasiOptions; +import run.endive.wasi.WasiPreview1; +import run.endive.wasm.WasmModule; + +public final class ComponentValidate { + + private ComponentValidate() {} + + private static final Logger logger = + new SystemLogger() { + @Override + public boolean isLoggable(Logger.Level level) { + return false; + } + }; + + private static final WasmModule MODULE = WasmToolsModule.load(); + + public static void validate(InputStream is) { + try (var stdinStream = new ByteArrayInputStream(new byte[0]); + var stdoutStream = new ByteArrayOutputStream(); + var stderrStream = new ByteArrayOutputStream(); + FileSystem fs = + ZeroFs.newFileSystem( + Configuration.unix().toBuilder() + .setAttributeViews("unix") + .build())) { + + Path inputDir = fs.getPath("input"); + Files.createDirectory(inputDir); + Path inputFile = inputDir.resolve("input.wasm"); + Files.write(inputFile, is.readAllBytes()); + + var options = + WasiOptions.builder() + .withStdin(stdinStream, false) + .withStdout(stdoutStream, false) + .withStderr(stderrStream, false) + .withDirectory(inputDir.toString(), inputDir) + .withArguments(List.of("wasm-tools", "validate", inputFile.toString())) + .build(); + + try (var wasi = + WasiPreview1.builder().withLogger(logger).withOptions(options).build()) { + var imports = ImportValues.builder().addFunction(wasi.toHostFunctions()).build(); + + try { + Instance.builder(MODULE) + .withMachineFactory(WasmToolsModule::create) + .withMemoryFactory(ByteArrayMemory::new) + .withImportValues(imports) + .build(); + } catch (WasiExitException e) { + if (e.exitCode() != 0) { + throw new ComponentValidateException( + stdoutStream.toString(StandardCharsets.UTF_8) + + stderrStream.toString(StandardCharsets.UTF_8), + e); + } + } + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/wasm-tools/src/main/java/run/endive/cm/tools/ComponentValidateException.java b/wasm-tools/src/main/java/run/endive/cm/tools/ComponentValidateException.java new file mode 100644 index 0000000..a566e90 --- /dev/null +++ b/wasm-tools/src/main/java/run/endive/cm/tools/ComponentValidateException.java @@ -0,0 +1,13 @@ +package run.endive.cm.tools; + +public class ComponentValidateException extends RuntimeException { + public ComponentValidateException() {} + + public ComponentValidateException(String message) { + super(message); + } + + public ComponentValidateException(String message, Throwable cause) { + super(message, cause); + } +}