diff --git a/.ctags b/.ctags new file mode 100644 index 000000000000..3c63175a86ba --- /dev/null +++ b/.ctags @@ -0,0 +1,13 @@ +--exclude=target/* +--exclude=*/target/* +--exclude=docs/* +--exclude=examples/* +--exclude=integration_tests/* +--exclude=fuzz/* +--exclude=*.md +--exclude=*.yml +--exclude=*.txt +--exclude=tags +--exclude=*.sh +--exclude=scripts/* +--exclude=Makefile diff --git a/.github/workflows/libwasmer-build.yml b/.github/workflows/libwasmer-build.yml new file mode 100644 index 000000000000..4b7baa816caf --- /dev/null +++ b/.github/workflows/libwasmer-build.yml @@ -0,0 +1,26 @@ +name: libwasmer-build + +on: + push: + branches: + - master + pull_request: + +env: + CARGO_TERM_COLOR: always + +jobs: + build: + name: Build dynamic library for ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: ubuntu-18.04 + artifact_name: libwasmer_linux_amd64.so + make_target: capi-linux-amd64 + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Make + run: make ${{ matrix.make_target }} diff --git a/.github/workflows/libwasmer-release.yml b/.github/workflows/libwasmer-release.yml new file mode 100644 index 000000000000..ca42325504bf --- /dev/null +++ b/.github/workflows/libwasmer-release.yml @@ -0,0 +1,40 @@ +name: libwasmer-release + +on: + release: + types: + - created + + +env: + CARGO_TERM_COLOR: always + +jobs: + build: + name: Build dynamic library for ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: ubuntu-18.04 + artifact_name: libwasmer_linux_amd64.so + make_target: capi-linux-amd64 + - os: macos-11 + artifact_name: libwasmer_darwin_amd64.dylib + make_target: capi-osx-amd64 + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Make + run: make ${{ matrix.make_target }} + - name: Get the version + id: get_version + if: startsWith(github.ref, 'refs/tags/') + run: echo ::set-output name=VERSION::${GITHUB_REF#refs/tags/} + - name: Release + uses: softprops/action-gh-release@v1 + if: startsWith(github.ref, 'refs/tags/') + with: + name: ${{ steps.get_version.outputs.VERSION }}-linux-amd64 + files: | + target/release/${{ matrix.artifact_name }} diff --git a/.gitignore b/.gitignore index b1166e461a8d..1889b7497402 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ .gdb_history **/.vscode api-docs-repo/ +tags +.ycm* /.cargo_home/ /package/ /dist/ diff --git a/.travis.upstream.yml b/.travis.upstream.yml new file mode 100644 index 000000000000..cda6825ee75b --- /dev/null +++ b/.travis.upstream.yml @@ -0,0 +1,75 @@ +arch: + - arm64 + +language: rust +install: travis_retry +rust: + - nightly-2019-12-19 + +cache: + directories: + - /home/travis/.sccache/ + - /home/travis/.cargo/bin/ + +script: + # Sccache + # - curl -L https://github.com/mozilla/sccache/releases/download/0.2.10/sccache-0.2.10-x86_64-unknown-linux-musl.tar.gz | tar xzf - + # - export RUSTC_WRAPPER=`pwd`/sccache-0.2.10-x86_64-unknown-linux-musl/sccache + - test -f /home/travis/.cargo/bin/sccache || travis_retry cargo install sccache + - export RUSTC_WRAPPER=/home/travis/.cargo/bin/sccache + - mkdir -p /home/travis/.sccache/ + - export SCCACHE_DIR="/home/travis/.sccache/" + - SCCACHE_ERROR_LOG=`pwd`/sccache.log RUST_LOG=debug $RUSTC_WRAPPER --start-server + - $RUSTC_WRAPPER -s + + # Tests + - make spectests-singlepass + +before_deploy: + # Release + - make release-singlepass + - mkdir -p artifacts + # Make capi + ## Disable capi tests for now: + ## They are failing because trampolines are not yet implemented for ARM + # - make test-capi-singlepass + - make capi-singlepass + - make build-capi-package + - cp ./wasmer-c-api.tar.gz ./artifacts/$(./scripts/capi-name.sh) + # Build WAPM + - make build-wapm + # Make package + - make build-install-package + - cp ./wasmer.tar.gz ./artifacts/$(./scripts/binary-name.sh) + +# before_deploy: +# # Set up git user name and tag this commit +# - git config --local user.name "Syrus Akbary" +# - git config --local user.email "syrus@wasmer.io" +# - export TRAVIS_TAG="0.10.2" +# # - git tag $TRAVIS_TAG + +deploy: + provider: releases + file_glob: true + file: artifacts/* + api_key: $GITHUB_OAUTH_TOKEN + # This is set to the previous artifacts are not deleted by travis + skip_cleanup: true + on: + tags: true + # branch: feature/singlepass-aarch64 + +addons: + apt: + packages: + - cmake + +branches: + only: + - master + - staging + - trying + # Making sure Travis runs on new Tags + - /^\d+\.\d+(\.\d+)?(-\S*)?$/ + - /^v\d+\.\d+(\.\d+)?(-\S*)?$/ diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000000..60c406ce151d --- /dev/null +++ b/.travis.yml @@ -0,0 +1,65 @@ +os: + - linux + - osx +arch: + - amd64 + - arm64 +osx_image: xcode11.3 + +language: rust +before_install: +- if [[ $TRAVIS_OS_NAME == linux ]]; then sudo apt-get install -y patchelf; fi +install: travis_retry +rust: + - nightly-2021-01-25 + +cache: + directories: + - $HOME/.sccache/ + - $HOME/.cargo/bin/ + +script: + - test -f $HOME/.cargo/bin/sccache || travis_retry cargo install sccache + - export RUSTC_WRAPPER=$HOME/.cargo/bin/sccache + - mkdir -p $HOME/.sccache/ + - export SCCACHE_DIR="$HOME/.sccache/" + - SCCACHE_ERROR_LOG=`pwd`/sccache.log RUST_LOG=debug $RUSTC_WRAPPER --start-server + - $RUSTC_WRAPPER -s + + - make capi + - ls ./target/release + - mkdir artifacts + + - if [[ "$TRAVIS_OS_NAME" == osx ]]; + then + mv ./target/release/libwasmer_runtime_c_api.dylib ./artifacts/libwasmer_darwin_amd64.dylib; + install_name_tool -id @executable_path/libwasmer_darwin_amd64.dylib ./artifacts/libwasmer_darwin_amd64.dylib; + fi + - if [[ "$TRAVIS_OS_NAME" == linux && "$TRAVIS_CPU_ARCH" == amd64 ]]; + then + mv ./target/release/libwasmer_runtime_c_api.so ./artifacts/libwasmer_linux_amd64.so; + patchelf --set-soname libwasmer_linux_amd64.so ./artifacts/libwasmer_linux_amd64.so; + fi + - if [[ "$TRAVIS_OS_NAME" == linux && "$TRAVIS_CPU_ARCH" == arm64 ]]; + then + mv ./target/release/libwasmer_runtime_c_api.so ./artifacts/libwasmer_linux_arm64.so; + patchelf --set-soname libwasmer_linux_arm64.so ./artifacts/libwasmer_linux_arm64.so; + fi + +deploy: + provider: releases + file_glob: true + file: artifacts/* + api_key: $GITHUB_OAUTH_TOKEN + skip_cleanup: true + draft: true + +addons: + apt: + packages: + - cmake + +branches: + only: + - master + - development diff --git a/Cargo.lock b/Cargo.lock index bd6efd3a3e30..73a1b5ca2c84 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -204,6 +204,27 @@ dependencies = [ "serde", ] +[[package]] +name = "bytecheck" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "314889ea31cda264cb7c3d6e6e5c9415a987ecb0e72c17c00d36fbb881d34abe" +dependencies = [ + "bytecheck_derive", + "ptr_meta", +] + +[[package]] +name = "bytecheck_derive" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a2b3b92c135dae665a6f760205b89187638e83bed17ef3e44e83c712cf30600" +dependencies = [ + "proc-macro2 1.0.20", + "quote 1.0.7", + "syn 1.0.40", +] + [[package]] name = "build-deps" version = "0.1.4" @@ -307,6 +328,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + [[package]] name = "clang-sys" version = "1.3.1" @@ -457,6 +484,7 @@ dependencies = [ "regalloc", "smallvec", "target-lexicon 0.12.3", + "thiserror", ] [[package]] @@ -1027,6 +1055,15 @@ dependencies = [ "ahash 0.7.6", ] +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +dependencies = [ + "ahash", +] + [[package]] name = "heck" version = "0.3.3" @@ -1851,6 +1888,41 @@ dependencies = [ "winapi", ] +[[package]] +name = "rend" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1033f6fe7ce48c8333e5412891b933e85d6a3a09728c4883240edf64e7a6f11a" +dependencies = [ + "bytecheck", +] + +[[package]] +name = "rkyv" +version = "0.7.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66bf572c17c77322f4d858c214def56b13a3c32b8d833cd6d28a92de8325ac5f" +dependencies = [ + "bytecheck", + "hashbrown", + "indexmap", + "ptr_meta", + "rend", + "rkyv_derive", + "seahash", +] + +[[package]] +name = "rkyv_derive" +version = "0.7.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3eca50f172b8e59e2080810fb41b65f047960c197149564d4bd0680af1888e" +dependencies = [ + "proc-macro2 1.0.20", + "quote 1.0.7", + "syn 1.0.40", +] + [[package]] name = "rend" version = "0.3.6" @@ -2000,6 +2072,12 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + [[package]] name = "semver" version = "0.9.0" @@ -2744,6 +2822,12 @@ dependencies = [ "leb128", ] +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + [[package]] name = "wasm-smith" version = "0.4.5" @@ -2981,7 +3065,7 @@ name = "wasmer-emscripten" version = "2.2.1" dependencies = [ "byteorder", - "getrandom", + "getrandom 0.1.14", "lazy_static", "libc", "log", @@ -3172,7 +3256,7 @@ dependencies = [ "bincode", "cfg-if 1.0.0", "generational-arena", - "getrandom", + "getrandom 0.1.14", "libc", "serde", "thiserror", @@ -3268,6 +3352,11 @@ version = "0.78.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52144d4c78e5cf8b055ceab8e5fa22814ce4315d6002ad32cfd914f37c12fd65" +[[package]] +name = "wasmparser" +version = "0.51.4" +source = "git+https://github.com/ElrondNetwork/wasmparser.rs#f776d0ae8c349463b25d023ab6c8b1a80a761fd7" + [[package]] name = "wasmparser" version = "0.82.0" diff --git a/Cargo.toml b/Cargo.toml index f1273bf12264..777afc874325 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ name = "wasmer-workspace" version = "2.2.1" description = "Wasmer workspace" authors = ["Wasmer Engineering Team "] -repository = "https://github.com/wasmerio/wasmer" +repository = "https://github.com/ElrondNetwork/wasmer" license = "MIT" edition = "2018" publish = false @@ -30,15 +30,11 @@ cfg-if = "1.0" [workspace] members = [ - "lib/api", - "lib/cache", - "lib/c-api", - "lib/cli", - "lib/compiler", - "lib/compiler-cranelift", - "lib/compiler-singlepass", - "lib/compiler-llvm", - "lib/derive", + # "lib/clif-backend", + "lib/singlepass-backend", + "lib/runtime", + "lib/runtime-core", + "lib/runtime-core-tests", "lib/emscripten", "lib/engine", "lib/engine-universal", @@ -294,3 +290,29 @@ required-features = ["cranelift"] name = "features" path = "examples/features.rs" required-features = ["cranelift"] + +[profile.dev] +opt-level = 0 # controls the `--opt-level` the compiler builds with. + # 0-1 is good for debugging. 2 is well-optimized. Max is 3. + # 's' attempts to reduce size, 'z' reduces size even more. +debug = true # (u32 or bool) Include debug information (debug symbols). + # Equivalent to `-C debuginfo=2` compiler flag. +rpath = false # controls whether compiler should set loader paths. + # If true, passes `-C rpath` flag to the compiler. +lto = false # Link Time Optimization usually reduces size of binaries + # and static libraries. Increases compilation time. + # If true, passes `-C lto` flag to the compiler, and if a + # string is specified like 'thin' then `-C lto=thin` will + # be passed. +debug-assertions = true # controls whether debug assertions are enabled + # (e.g., debug_assert!() and arithmetic overflow checks) +codegen-units = 16 # if > 1 enables parallel code generation which improves + # compile times, but prevents some optimizations. + # Passes `-C codegen-units`. +panic = 'unwind' # panic strategy (`-C panic=...`), can also be 'abort' +incremental = true # whether or not incremental compilation is enabled + # This can be overridden globally with the CARGO_INCREMENTAL + # environment variable or `build.incremental` config + # variable. Incremental is only used for path sources. +overflow-checks = true # use overflow checks for integer arithmetic. + # Passes the `-C overflow-checks=...` flag to the compiler. diff --git a/Makefile b/Makefile index 00b2a1c1c1d4..eae6422252b4 100644 --- a/Makefile +++ b/Makefile @@ -106,7 +106,7 @@ ENABLE_LLVM ?= ENABLE_SINGLEPASS ?= # Which compilers we build. These have dependencies that may not be on the system. -compilers := +compilers := ## # Cranelift @@ -276,7 +276,7 @@ comma := , # Define the compiler Cargo features for all crates. compiler_features := --features $(subst $(space),$(comma),$(compilers)) -capi_compilers_engines_exclude := +capi_compilers_engines_exclude := # Define the compiler Cargo features for the C API. It always excludes # LLVM for the moment because it causes the linker to fail since LLVM is not statically linked. diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 000000000000..2c40ca4fb32a --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,335 @@ +name: $(Build.SourceBranch)-$(date:yyyyMMdd)$(rev:.r) + +# The Different jobs (lint, test, build to run) +jobs: + - job: changelog + steps: + - bash: | + git --no-pager diff --name-only HEAD $(git merge-base HEAD master) --exit-code CHANGELOG.md + displayName: Changelog Updated + + - job: lint + pool: + vmImage: "macos-10.14" + steps: + - checkout: self + - template: .azure/install-rust.yml + - script: | + rustup component add rustfmt + displayName: Lint dependencies + - script: cargo fmt --all -- --check + displayName: Lint + variables: + rust_toolchain: '1.40.0' + + - job: clippy_lint + pool: + vmImage: "ubuntu-16.04" + steps: + - checkout: self + - template: .azure/install-rust.yml + - template: .azure/install-llvm.yml + - template: .azure/install-sccache.yml + - template: .azure/install-cmake.yml + - script: | + rustup component add rustfmt + rustup component add clippy || cargo install --git https://github.com/rust-lang/rust-clippy/ --force clippy + displayName: Lint dependencies with clippy + - script: cargo clippy --workspace + displayName: Clippy Lint + variables: + rust_toolchain: nightly-2019-12-19 + + - job: Test + strategy: + matrix: + linux: + imageName: "ubuntu-16.04" + rust_toolchain: nightly-2019-12-19 + mac: + imageName: "macos-10.14" + rust_toolchain: nightly-2019-12-19 + # By default schannel checks revocation of certificates unlike some other SSL + # backends, but we've historically had problems on CI where a revocation + # server goes down presumably. See #43333 for more info + CARGO_HTTP_CHECK_REVOKE: false + windows: + imageName: "vs2017-win2016" + rust_toolchain: '1.40.0' + pool: + vmImage: $(imageName) + condition: in(variables['Build.SourceBranch'], 'refs/heads/master', 'refs/heads/staging', 'refs/heads/trying') + steps: + - checkout: self + - template: .azure/install-rust.yml + - template: .azure/install-llvm.yml + - template: .azure/install-sccache.yml + - template: .azure/install-cmake.yml + - bash: | + hostname + uname -a + displayName: System info (*nix) + condition: and(succeeded(), not(eq(variables['Agent.OS'], 'Windows_NT'))) + - bash: | + cat /proc/cpuinfo + cat /proc/meminfo + displayName: System info - Extended (Linux) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) + - bash: | + sysctl -a | grep machdep.cpu + displayName: System info - Extended (Mac) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) + - bash: make test + displayName: Tests (*nix) + condition: and(succeeded(), not(eq(variables['Agent.OS'], 'Windows_NT'))) + - bash: make spectests-cranelift + displayName: Tests (Windows) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) + + - job: Check + pool: + vmImage: "ubuntu-16.04" + variables: + rust_toolchain: nightly-2019-12-19 + condition: in(variables['Build.SourceBranch'], 'refs/heads/master', 'refs/heads/staging', 'refs/heads/trying') + steps: + - checkout: self + - template: .azure/install-rust.yml + - template: .azure/install-llvm.yml + - template: .azure/install-sccache.yml + - template: .azure/install-cmake.yml + - bash: make check + displayName: Check with Flags + condition: and(succeeded(), not(eq(variables['Agent.OS'], 'Windows_NT'))) + + - job: Build_CLI + strategy: + matrix: + linux: + imageName: "ubuntu-16.04" + rust_toolchain: nightly-2019-12-19 + mac: + imageName: "macos-10.14" + rust_toolchain: nightly-2019-12-19 + MACOSX_DEPLOYMENT_TARGET: 10.10 + windows: + imageName: "vs2017-win2016" + rust_toolchain: '1.40.0' + # RUSTFLAGS: -Ctarget-feature=+crt-static + pool: + vmImage: $(imageName) + condition: | + or( + in(variables['Build.SourceBranch'], 'refs/heads/master', 'refs/heads/staging', 'refs/heads/trying'), + startsWith(variables['Build.SourceBranch'], 'refs/tags') + ) + steps: + - checkout: self + - template: .azure/install-rust.yml + - template: .azure/install-llvm.yml + - template: .azure/install-sccache.yml + - template: .azure/install-cmake.yml + - template: .azure/install-innosetup.yml + - bash: | + mkdir -p artifacts + displayName: Create Artifacts Dir + - bash: make release + displayName: Build (*nix) + condition: and(succeeded(), not(eq(variables['Agent.OS'], 'Windows_NT'))) + - bash: sudo apt-get install musl-tools && make release-musl + displayName: Build (Linux, x86_64-unknown-linux-musl) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) + - bash: make release-llvm + displayName: Build (Windows) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) + - bash: | + make build-wapm + displayName: Build WAPM + condition: | + startsWith(variables['Build.SourceBranch'], 'refs/tags') + - bash: | + make build-install-package + cp ./wasmer.tar.gz ./artifacts/$(./scripts/binary-name.sh) + displayName: Build Distribution (*nix) + condition: | + and( + succeeded(), + startsWith(variables['Build.SourceBranch'], 'refs/tags'), + not(eq(variables['Agent.OS'], 'Windows_NT')) + ) + - bash: | + cd ./src/installer + iscc wasmer.iss + cp WasmerInstaller.exe ../../artifacts/wasmer-windows.exe + displayName: Build Distribution (Windows) + condition: | + and( + succeeded(), + startsWith(variables['Build.SourceBranch'], 'refs/tags'), + eq(variables['Agent.OS'], 'Windows_NT') + ) + - publish: $(System.DefaultWorkingDirectory)/artifacts + artifact: cli-$(Agent.OS) + + - job: Build_Library + strategy: + matrix: + linux: + imageName: "ubuntu-16.04" + rust_toolchain: nightly-2019-12-19 + mac: + imageName: "macos-10.14" + rust_toolchain: nightly-2019-12-19 + MACOSX_DEPLOYMENT_TARGET: 10.10 + windows: + imageName: "vs2017-win2016" + rust_toolchain: '1.40.0' + # RUSTFLAGS: -Ctarget-feature=+crt-static + pool: + vmImage: $(imageName) + condition: | + or( + in(variables['Build.SourceBranch'], 'refs/heads/master', 'refs/heads/staging', 'refs/heads/trying'), + startsWith(variables['Build.SourceBranch'], 'refs/tags') + ) + steps: + - checkout: self + - template: .azure/install-rust.yml + - template: .azure/install-llvm.yml + - template: .azure/install-sccache.yml + - template: .azure/install-cmake.yml + - bash: | + mkdir -p artifacts + displayName: Create Artifacts Dir + - bash: | + make test-capi + displayName: Test c-api + condition: and(succeeded(), not(eq(variables['Agent.OS'], 'Windows_NT'))) + - bash: | + make capi + displayName: Build c-api + - bash: | + make build-capi-package + cp ./wasmer-c-api.tar.gz ./artifacts/$(./scripts/capi-name.sh) + displayName: Build c-api artifacts (Unix) + condition: | + and( + succeeded(), + not(eq(variables['Agent.OS'], 'Windows_NT')) + ) + - bash: | + make build-capi-package + cp ./wasmer-c-api.tar.gz ./artifacts/wasmer-c-api-windows.tar.gz + displayName: Build c-api artifacts (Windows) + condition: | + and( + succeeded(), + eq(variables['Agent.OS'], 'Windows_NT') + ) + - publish: $(System.DefaultWorkingDirectory)/artifacts + artifact: library-$(Agent.OS) + + - job: Build_Docs + pool: + vmImage: "ubuntu-16.04" + variables: + rust_toolchain: nightly-2019-12-19 + condition: in(variables['Build.SourceBranch'], 'refs/heads/master', 'refs/heads/staging', 'refs/heads/trying') + steps: + - checkout: self + - template: .azure/install-rust.yml + - template: .azure/install-llvm.yml + - template: .azure/install-sccache.yml + - template: .azure/install-cmake.yml + - bash: | + sudo apt-get install doxygen graphviz + displayName: Install doxygen + - bash: | + make docs + displayName: Build docs + - publish: $(System.DefaultWorkingDirectory)/api-docs + artifact: api-docs + displayName: Save Docs artifact + + - job: Publish + dependsOn: + - Build_CLI + - Build_Library + condition: | + startsWith(variables['Build.SourceBranch'], 'refs/tags') + steps: + # - download: current + - task: DownloadPipelineArtifact@1 + inputs: + targetPath: $(Build.ArtifactStagingDirectory) + - bash: | + ls $ARTIFACT_STAGING_DIRECTORY + displayName: List Artifacts + env: + ARTIFACT_STAGING_DIRECTORY: $(Build.ArtifactStagingDirectory) + - script: VERSION_TAG=`git describe --tags` && echo "##vso[task.setvariable variable=VERSION_TAG]$VERSION_TAG" + displayName: Set the tag name as an environment variable + - task: GithubRelease@0 + displayName: "Create GitHub Release" + condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags')) + continueOnError: true + inputs: + gitHubConnection: 'wasmer' + repositoryName: 'ElrondNetwork/wasmer' + action: 'create' + target: '$(Build.SourceVersion)' + title: '$(VERSION_TAG)' + addChangeLog: false + tagSource: 'auto' + # TODO: automate it all by getting the release notes from somewhere else and using the `releaseNotesFile` key + isDraft: false + isPreRelease: false + - task: GithubRelease@0 + displayName: "Update GitHub Release with assets" + condition: and(succeededOrFailed(), startsWith(variables['Build.SourceBranch'], 'refs/tags')) + inputs: + gitHubConnection: 'wasmer' + repositoryName: 'wasmerio/wasmer' + action: 'edit' + target: '$(Build.SourceVersion)' + title: '$(VERSION_TAG)' + tag: $(VERSION_TAG) + addChangeLog: false + tagSource: 'auto' + # TODO: automate it all by getting the release notes from somewhere else and using the `releaseNotesFile` key + isDraft: false + isPreRelease: false + assets: '$(Build.ArtifactStagingDirectory)/**' + + - job: Publish_Docs + dependsOn: + - Build_Docs + displayName: Deploy API Documentation to GitHub + pool: + vmImage: "ubuntu-16.04" + condition: in(variables['Build.SourceBranch'], 'refs/heads/master') + steps: + - checkout: self + - task: DownloadPipelineArtifact@2 + inputs: + artifactName: api-docs + targetPath: $(System.DefaultWorkingDirectory)/api-docs + - bash: | + git config --global user.email "bot@wasmer.io" + git config --global user.name "wasmerbot" + make docs-publish + env: + RUST_DOCS_DIR: $(Pipeline.Workspace)/api-docs + GITHUB_DOCS_TOKEN: $(GITHUB_DOCS_TOKEN) + SOURCE_VERSION: $(Build.SourceVersion) + +# We only run the pipelines on PRs to Master +pr: + - master + +# Otherwise, we test in any of these branches (master or bors related) +trigger: + - master + - staging + - trying + - refs/tags/* diff --git a/examples/parallel/Cargo.toml b/examples/parallel/Cargo.toml new file mode 100644 index 000000000000..2cb22900a430 --- /dev/null +++ b/examples/parallel/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "parallel" +version = "0.1.0" +authors = ["The Wasmer Engineering Team "] +edition = "2018" +repository = "https://github.com/ElrondNetwork/wasmer" +publish = false +license = "MIT" + +[dependencies] +rayon = "1.2" +time = "0.1" +wasmer-runtime = { path = "../../lib/runtime" } +wasmer-runtime-core = { path = "../../lib/runtime-core" } diff --git a/examples/plugin-for-example/wapm.toml b/examples/plugin-for-example/wapm.toml new file mode 100644 index 000000000000..b4fd2e2f3582 --- /dev/null +++ b/examples/plugin-for-example/wapm.toml @@ -0,0 +1,12 @@ +[package] +name = "plugin-for-example" +version = "0.1.0" +description = "A plugin for our example system" +readme = "README.md" +repository = "https://github.com/ElrondNetwork/wasmer/examples/plugin-for-example" +license = "MIT" + +[[module]] +name = "plugin-for-example" +source = "../../target/wasm32-unknown-wasi/release/plugin-for-example.wasm" +abi = "none" diff --git a/instructions.txt b/instructions.txt new file mode 100644 index 000000000000..b240bbd8872f --- /dev/null +++ b/instructions.txt @@ -0,0 +1,410 @@ +Unreachable +Nop +Block +Loop +If +Else +End +Br +BrIf +BrTable +Return +Call +CallIndirect +Drop +Select +GetLocal +SetLocal +TeeLocal +GetGlobal +SetGlobal +I32Load +I64Load +F32Load +F64Load +I32Load8S +I32Load8U +I32Load16S +I32Load16U +I64Load8S +I64Load8U +I64Load16S +I64Load16U +I64Load32S +I64Load32U +I32Store +I64Store +F32Store +F64Store +I32Store8 +I32Store16 +I64Store8 +I64Store16 +I64Store32 +MemorySize +MemoryGrow +I32Const +I64Const +F32Const +F64Const +RefNull +RefIsNull +I32Eqz +I32Eq +I32Ne +I32LtS +I32LtU +I32GtS +I32GtU +I32LeS +I32LeU +I32GeS +I32GeU +I64Eqz +I64Eq +I64Ne +I64LtS +I64LtU +I64GtS +I64GtU +I64LeS +I64LeU +I64GeS +I64GeU +F32Eq +F32Ne +F32Lt +F32Gt +F32Le +F32Ge +F64Eq +F64Ne +F64Lt +F64Gt +F64Le +F64Ge +I32Clz +I32Ctz +I32Popcnt +I32Add +I32Sub +I32Mul +I32DivS +I32DivU +I32RemS +I32RemU +I32And +I32Or +I32Xor +I32Shl +I32ShrS +I32ShrU +I32Rotl +I32Rotr +I64Clz +I64Ctz +I64Popcnt +I64Add +I64Sub +I64Mul +I64DivS +I64DivU +I64RemS +I64RemU +I64And +I64Or +I64Xor +I64Shl +I64ShrS +I64ShrU +I64Rotl +I64Rotr +F32Abs +F32Neg +F32Ceil +F32Floor +F32Trunc +F32Nearest +F32Sqrt +F32Add +F32Sub +F32Mul +F32Div +F32Min +F32Max +F32Copysign +F64Abs +F64Neg +F64Ceil +F64Floor +F64Trunc +F64Nearest +F64Sqrt +F64Add +F64Sub +F64Mul +F64Div +F64Min +F64Max +F64Copysign +I32WrapI64 +I32TruncSF32 +I32TruncUF32 +I32TruncSF64 +I32TruncUF64 +I64ExtendSI32 +I64ExtendUI32 +I64TruncSF32 +I64TruncUF32 +I64TruncSF64 +I64TruncUF64 +F32ConvertSI32 +F32ConvertUI32 +F32ConvertSI64 +F32ConvertUI64 +F32DemoteF64 +F64ConvertSI32 +F64ConvertUI32 +F64ConvertSI64 +F64ConvertUI64 +F64PromoteF32 +I32ReinterpretF32 +I64ReinterpretF64 +F32ReinterpretI32 +F64ReinterpretI64 +I32Extend8S +I32Extend16S +I64Extend8S +I64Extend16S +I64Extend32S +I32TruncSSatF32 +I32TruncUSatF32 +I32TruncSSatF64 +I32TruncUSatF64 +I64TruncSSatF32 +I64TruncUSatF32 +I64TruncSSatF64 +I64TruncUSatF64 +MemoryInit +DataDrop +MemoryCopy +MemoryFill +TableInit +ElemDrop +TableCopy +TableGet +TableSet +TableGrow +TableSize +Wake +I32Wait +I64Wait +Fence +I32AtomicLoad +I64AtomicLoad +I32AtomicLoad8U +I32AtomicLoad16U +I64AtomicLoad8U +I64AtomicLoad16U +I64AtomicLoad32U +I32AtomicStore +I64AtomicStore +I32AtomicStore8 +I32AtomicStore16 +I64AtomicStore8 +I64AtomicStore16 +I64AtomicStore32 +I32AtomicRmwAdd +I64AtomicRmwAdd +I32AtomicRmw8UAdd +I32AtomicRmw16UAdd +I64AtomicRmw8UAdd +I64AtomicRmw16UAdd +I64AtomicRmw32UAdd +I32AtomicRmwSub +I64AtomicRmwSub +I32AtomicRmw8USub +I32AtomicRmw16USub +I64AtomicRmw8USub +I64AtomicRmw16USub +I64AtomicRmw32USub +I32AtomicRmwAnd +I64AtomicRmwAnd +I32AtomicRmw8UAnd +I32AtomicRmw16UAnd +I64AtomicRmw8UAnd +I64AtomicRmw16UAnd +I64AtomicRmw32UAnd +I32AtomicRmwOr +I64AtomicRmwOr +I32AtomicRmw8UOr +I32AtomicRmw16UOr +I64AtomicRmw8UOr +I64AtomicRmw16UOr +I64AtomicRmw32UOr +I32AtomicRmwXor +I64AtomicRmwXor +I32AtomicRmw8UXor +I32AtomicRmw16UXor +I64AtomicRmw8UXor +I64AtomicRmw16UXor +I64AtomicRmw32UXor +I32AtomicRmwXchg +I64AtomicRmwXchg +I32AtomicRmw8UXchg +I32AtomicRmw16UXchg +I64AtomicRmw8UXchg +I64AtomicRmw16UXchg +I64AtomicRmw32UXchg +I32AtomicRmwCmpxchg +I64AtomicRmwCmpxchg +I32AtomicRmw8UCmpxchg +I32AtomicRmw16UCmpxchg +I64AtomicRmw8UCmpxchg +I64AtomicRmw16UCmpxchg +I64AtomicRmw32UCmpxchg +V128Load +V128Store +V128Const +I8x16Splat +I8x16ExtractLaneS +I8x16ExtractLaneU +I8x16ReplaceLane +I16x8Splat +I16x8ExtractLaneS +I16x8ExtractLaneU +I16x8ReplaceLane +I32x4Splat +I32x4ExtractLane +I32x4ReplaceLane +I64x2Splat +I64x2ExtractLane +I64x2ReplaceLane +F32x4Splat +F32x4ExtractLane +F32x4ReplaceLane +F64x2Splat +F64x2ExtractLane +F64x2ReplaceLane +I8x16Eq +I8x16Ne +I8x16LtS +I8x16LtU +I8x16GtS +I8x16GtU +I8x16LeS +I8x16LeU +I8x16GeS +I8x16GeU +I16x8Eq +I16x8Ne +I16x8LtS +I16x8LtU +I16x8GtS +I16x8GtU +I16x8LeS +I16x8LeU +I16x8GeS +I16x8GeU +I32x4Eq +I32x4Ne +I32x4LtS +I32x4LtU +I32x4GtS +I32x4GtU +I32x4LeS +I32x4LeU +I32x4GeS +I32x4GeU +F32x4Eq +F32x4Ne +F32x4Lt +F32x4Gt +F32x4Le +F32x4Ge +F64x2Eq +F64x2Ne +F64x2Lt +F64x2Gt +F64x2Le +F64x2Ge +V128Not +V128And +V128Or +V128Xor +V128Bitselect +I8x16Neg +I8x16AnyTrue +I8x16AllTrue +I8x16Shl +I8x16ShrS +I8x16ShrU +I8x16Add +I8x16AddSaturateS +I8x16AddSaturateU +I8x16Sub +I8x16SubSaturateS +I8x16SubSaturateU +I8x16Mul +I16x8Neg +I16x8AnyTrue +I16x8AllTrue +I16x8Shl +I16x8ShrS +I16x8ShrU +I16x8Add +I16x8AddSaturateS +I16x8AddSaturateU +I16x8Sub +I16x8SubSaturateS +I16x8SubSaturateU +I16x8Mul +I32x4Neg +I32x4AnyTrue +I32x4AllTrue +I32x4Shl +I32x4ShrS +I32x4ShrU +I32x4Add +I32x4Sub +I32x4Mul +I64x2Neg +I64x2AnyTrue +I64x2AllTrue +I64x2Shl +I64x2ShrS +I64x2ShrU +I64x2Add +I64x2Sub +F32x4Abs +F32x4Neg +F32x4Sqrt +F32x4Add +F32x4Sub +F32x4Mul +F32x4Div +F32x4Min +F32x4Max +F64x2Abs +F64x2Neg +F64x2Sqrt +F64x2Add +F64x2Sub +F64x2Mul +F64x2Div +F64x2Min +F64x2Max +I32x4TruncSF32x4Sat +I32x4TruncUF32x4Sat +I64x2TruncSF64x2Sat +I64x2TruncUF64x2Sat +F32x4ConvertSI32x4 +F32x4ConvertUI32x4 +F64x2ConvertSI64x2 +F64x2ConvertUI64x2 +V8x16Swizzle +V8x16Shuffle +I8x16LoadSplat +I16x8LoadSplat +I32x4LoadSplat +I64x2LoadSplat diff --git a/lib/clif-backend/Cargo.toml b/lib/clif-backend/Cargo.toml new file mode 100644 index 000000000000..c186bd37dc4f --- /dev/null +++ b/lib/clif-backend/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "wasmer-clif-backend" +version = "0.15.0" +description = "Wasmer runtime Cranelift compiler backend" +license = "MIT" +authors = ["The Wasmer Engineering Team "] +repository = "https://github.com/ElrondNetwork/wasmer" +keywords = ["wasm", "webassembly", "compiler", "JIT", "AOT"] +categories = ["wasm"] +edition = "2018" +readme = "README.md" + +[dependencies] +wasmer-runtime-core = { path = "../runtime-core", version = "0.15.0" } +cranelift-native = "0.59.0" +cranelift-codegen = "0.59.0" +cranelift-entity = "0.59.0" +cranelift-frontend = { package = "wasmer-clif-fork-frontend", version = "0.59.0" } +cranelift-wasm = { package = "wasmer-clif-fork-wasm", version = "0.59.0" } +target-lexicon = "0.10" +wasmparser = { git = "https://github.com/ElrondNetwork/wasmparser.rs" } +byteorder = "1.3.2" +nix = "0.15.0" +libc = "0.2.60" +rayon = "1.1" +wasm-debug = { optional = true, version = "0.1" } + +# Dependencies for caching. +[dependencies.serde] +version = "1.0" +features = ["rc"] +[dependencies.serde_derive] +version = "1.0" +[dependencies.serde_bytes] +version = "0.11" +[dependencies.serde-bench] +version = "0.0.7" + +[target.'cfg(windows)'.dependencies] +winapi = { version = "0.3", features = ["errhandlingapi", "minwindef", "minwinbase", "winnt"] } +wasmer-win-exception-handler = { path = "../win-exception-handler", version = "0.15.0" } + +[features] +generate-debug-information = ["wasm-debug"] diff --git a/lib/clif-backend/src/lib.rs b/lib/clif-backend/src/lib.rs new file mode 100644 index 000000000000..25822c3c648a --- /dev/null +++ b/lib/clif-backend/src/lib.rs @@ -0,0 +1,71 @@ +//! The Wasmer Cranelift Backend crate is used to compile wasm binary code via parse events from the +//! Wasmer runtime common parser code into machine code. +//! + +#![deny( + dead_code, + nonstandard_style, + unused_imports, + unused_mut, + unused_variables, + unused_unsafe, + unreachable_patterns +)] +#![doc(html_favicon_url = "https://wasmer.io/static/icons/favicon.ico")] +#![doc(html_logo_url = "https://avatars3.githubusercontent.com/u/44205449?s=200&v=4")] + +mod cache; +mod code; +mod libcalls; +mod module; +mod relocation; +mod resolver; +mod signal; +mod trampoline; + +use cranelift_codegen::{ + isa, + settings::{self, Configurable}, +}; +use target_lexicon::Triple; + +#[macro_use] +extern crate serde_derive; + +extern crate rayon; +extern crate serde; + +fn get_isa() -> Box { + let flags = { + let mut builder = settings::builder(); + builder.set("opt_level", "speed_and_size").unwrap(); + builder.set("enable_jump_tables", "false").unwrap(); + + if cfg!(test) || cfg!(debug_assertions) { + builder.set("enable_verifier", "true").unwrap(); + } else { + builder.set("enable_verifier", "false").unwrap(); + } + + let flags = settings::Flags::new(builder); + debug_assert_eq!(flags.opt_level(), settings::OptLevel::SpeedAndSize); + flags + }; + isa::lookup(Triple::host()).unwrap().finish(flags) +} + +/// The current version of this crate +pub const VERSION: &str = env!("CARGO_PKG_VERSION"); + +use wasmer_runtime_core::codegen::SimpleStreamingCompilerGen; + +/// Streaming compiler implementation for the Cranelift backed. Compiles web assembly binary into +/// machine code. +pub type CraneliftCompiler = SimpleStreamingCompilerGen< + code::CraneliftModuleCodeGenerator, + code::CraneliftFunctionCodeGenerator, + signal::Caller, + code::CodegenError, +>; + +pub use code::CraneliftModuleCodeGenerator; diff --git a/lib/compiler-llvm/src/translator/stackmap.rs b/lib/compiler-llvm/src/translator/stackmap.rs index cb6abc48fb5f..9aa5fdda1e0e 100644 --- a/lib/compiler-llvm/src/translator/stackmap.rs +++ b/lib/compiler-llvm/src/translator/stackmap.rs @@ -253,8 +253,8 @@ impl StackmapEntry { machine_stack_layout.push(major.clone()); } else { machine_stack_layout.push(MachineValue::TwoHalves(Box::new(( - major.clone(), - minor.clone(), + major.clone().into(), + minor.clone().into(), )))); } } diff --git a/lib/dev-utils/Cargo.toml b/lib/dev-utils/Cargo.toml new file mode 100644 index 000000000000..477f26ba07db --- /dev/null +++ b/lib/dev-utils/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "wasmer-dev-utils" +version = "0.15.0" +description = "Wasmer runtime core library" +license = "MIT" +authors = ["The Wasmer Engineering Team "] +edition = "2018" +repository = "https://github.com/ElrondNetwork/wasmer" +publish = false + +[dependencies] +libc = "0.2.60" diff --git a/lib/emscripten-tests/Cargo.toml b/lib/emscripten-tests/Cargo.toml new file mode 100644 index 000000000000..5ded07d0f1bb --- /dev/null +++ b/lib/emscripten-tests/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "wasmer-emscripten-tests" +version = "0.15.0" +description = "Tests for our Emscripten implementation" +license = "MIT" +authors = ["The Wasmer Engineering Team "] +edition = "2018" +publish = false +build = "build/mod.rs" + +[dependencies] +wasmer-emscripten = { path = "../emscripten", version = "0.15.0" } +wasmer-runtime = { path = "../runtime", version = "0.15.0", default-features = false } +wasmer-clif-backend = { path = "../clif-backend", version = "0.15.0", optional = true} +wasmer-llvm-backend = { path = "../llvm-backend", version = "0.15.0", optional = true, features = ["test"] } +wasmer-singlepass-backend = { path = "../singlepass-backend", version = "0.15.0", optional = true } + +[dev-dependencies] +wabt = "0.9.1" +wasmer-dev-utils = { path = "../dev-utils", version = "0.15.0"} + +[build-dependencies] +glob = "0.3" + +[features] +# clif = ["wasmer-clif-backend", "wasmer-runtime/default-backend-cranelift"] +singlepass = ["wasmer-singlepass-backend", "wasmer-runtime/default-backend-singlepass"] +llvm = ["wasmer-llvm-backend", "wasmer-runtime/default-backend-llvm"] diff --git a/lib/emscripten/Cargo.toml b/lib/emscripten/Cargo.toml index 182f782ba7cd..f775e0367d89 100644 --- a/lib/emscripten/Cargo.toml +++ b/lib/emscripten/Cargo.toml @@ -5,7 +5,7 @@ description = "Emscripten implementation library for Wasmer WebAssembly runtime" categories = ["wasm", "os"] keywords = ["wasm", "webassembly", "abi", "emscripten", "posix"] authors = ["Wasmer Engineering Team "] -repository = "https://github.com/wasmerio/wasmer" +repository = "https://github.com/ElrondNetwork/wasmer" license = "MIT" readme = "README.md" edition = "2018" diff --git a/lib/llvm-backend/Cargo.toml b/lib/llvm-backend/Cargo.toml new file mode 100644 index 000000000000..5fa8665cdaad --- /dev/null +++ b/lib/llvm-backend/Cargo.toml @@ -0,0 +1,49 @@ +[package] +name = "wasmer-llvm-backend" +version = "0.15.0" +license = "MIT" +authors = ["The Wasmer Engineering Team "] +repository = "https://github.com/wasmerio/wasmer" +keywords = ["wasm", "webassembly", "compiler", "JIT", "llvm"] +categories = ["wasm"] +edition = "2018" +readme = "README.md" + +[dependencies] +wasmer-runtime-core = { path = "../runtime-core", version = "0.15.0" } +wasmparser = { git = "https://github.com/ElrondNetwork/wasmparser.rs" } +smallvec = "0.6" +goblin = "0.0.24" +libc = "0.2.60" +byteorder = "1" + +[target.'cfg(target_arch = "x86_64")'.dependencies.inkwell] +git = "https://github.com/TheDan64/inkwell" +rev = "0a864ebf68b33d4d514b67796264b03898aa0944" +default-features = false +features = ["llvm8-0", "target-x86"] + +[target.'cfg(target_arch = "aarch64")'.dependencies.inkwell] +git = "https://github.com/TheDan64/inkwell" +rev = "0a864ebf68b33d4d514b67796264b03898aa0944" +default-features = false +features = ["llvm8-0", "target-aarch64"] + +[target.'cfg(unix)'.dependencies] +nix = "0.15" + +[target.'cfg(windows)'.dependencies] +winapi = { version = "0.3", features = ["memoryapi"] } + +[build-dependencies] +cc = "1.0" +lazy_static = "1.4" +regex = "1.2" +semver = "0.9" +rustc_version = "0.2" + +[dev-dependencies] +wabt = "0.9.1" + +[features] +test = [] diff --git a/lib/middleware-common-tests/Cargo.toml b/lib/middleware-common-tests/Cargo.toml new file mode 100644 index 000000000000..d7d4aacd8fd5 --- /dev/null +++ b/lib/middleware-common-tests/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "wasmer-middleware-common-tests" +version = "0.15.0" +authors = ["The Wasmer Engineering Team "] +edition = "2018" +repository = "https://github.com/ElrondNetwork/wasmer" +license = "MIT" +publish = false + +[dependencies] +wasmer-runtime-core = { path = "../runtime-core", version = "0.15.0" } +wasmer-middleware-common = { path = "../middleware-common", version = "0.15.0" } +wasmer-clif-backend = { path = "../clif-backend", version = "0.15.0", optional = true } +wasmer-llvm-backend = { path = "../llvm-backend", version = "0.15.0", features = ["test"], optional = true } +wasmer-singlepass-backend = { path = "../singlepass-backend", version = "0.15.0", optional = true } + +[features] +clif = ["wasmer-clif-backend"] +llvm = ["wasmer-llvm-backend"] +singlepass = ["wasmer-singlepass-backend"] + +[dev-dependencies] +wabt = "0.9.1" +criterion = "0.2" + +[[bench]] +name = "metering_benchmark" +harness = false diff --git a/lib/middleware-common/Cargo.toml b/lib/middleware-common/Cargo.toml new file mode 100644 index 000000000000..d1c0ea4a09eb --- /dev/null +++ b/lib/middleware-common/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "wasmer-middleware-common" +repository = "https://github.com/ElrondNetwork/wasmer" +version = "0.15.0" +description = "Wasmer runtime common middlewares" +license = "MIT" +authors = ["The Wasmer Engineering Team "] +keywords = ["wasm", "webassembly", "middleware", "metering"] +categories = ["wasm"] +edition = "2018" + +[dependencies] +wasmer-runtime-core = { path = "../runtime-core", version = "0.15.0" } diff --git a/lib/middleware-common/src/lib.rs b/lib/middleware-common/src/lib.rs new file mode 100644 index 000000000000..e1be116f1554 --- /dev/null +++ b/lib/middleware-common/src/lib.rs @@ -0,0 +1,22 @@ +#![deny( + dead_code, + nonstandard_style, + unused_imports, + unused_mut, + unused_variables, + unused_unsafe, + unreachable_patterns +)] +#![doc(html_favicon_url = "https://wasmer.io/static/icons/favicon.ico")] +#![doc(html_logo_url = "https://avatars3.githubusercontent.com/u/44205449?s=200&v=4")] + +#[cfg(unix)] +pub mod block_trace; +pub mod call_trace; + +pub mod metering; +pub mod metering_costs; + +pub mod runtime_breakpoints; +pub mod opcode_trace; +pub mod opcode_control; diff --git a/lib/middleware-common/src/metering.rs b/lib/middleware-common/src/metering.rs new file mode 100644 index 000000000000..4678c7720e87 --- /dev/null +++ b/lib/middleware-common/src/metering.rs @@ -0,0 +1,164 @@ +use wasmer_runtime_core::{ + codegen::{Event, EventSink, FunctionMiddleware, InternalEvent}, + module::ModuleInfo, + vm::{Ctx, InternalField}, + wasmparser::{Operator, Type as WpType, TypeOrFuncType as WpTypeOrFuncType}, + Instance, +}; + +use crate::metering_costs::{get_opcode_index, get_local_allocate_cost_index}; +use crate::runtime_breakpoints::{push_runtime_breakpoint, BREAKPOINT_VALUE_OUT_OF_GAS}; + +static FIELD_USED_POINTS: InternalField = InternalField::allocate(); +static FIELD_POINTS_LIMIT: InternalField = InternalField::allocate(); + +/// Metering is a compiler middleware that calculates the cost of WebAssembly instructions at compile +/// time and will count the cost of executed instructions at runtime. Within the Metering functionality, +/// this instruction cost is called `points`. +/// +/// The Metering struct takes a `limit` parameter which is the maximum number of points which can be +/// used by an instance during a function call. If this limit is exceeded, the function call will +/// trap. Each instance has a `points_used` field which can be used to track points used during +/// a function call and should be set back to zero after a function call. +/// +/// Each compiler backend with Metering enabled should produce the same cost used at runtime for +/// the same function calls so we can say that the metering is deterministic. +/// + +pub struct Metering<'a> { + unmetered_locals: usize, + current_block: u64, + func_locals_costs: u32, + opcode_costs: &'a [u32], +} + +impl<'a> Metering<'a> { + pub fn new(opcode_costs: &'a [u32], unmetered_locals: usize) -> Metering<'a> { + Metering { + unmetered_locals, + current_block: 0, + func_locals_costs: 0, + opcode_costs, + } + } +} + +#[derive(Copy, Clone, Debug)] +pub struct ExecutionLimitExceededError; + +impl<'q> FunctionMiddleware for Metering<'q> { + type Error = String; + + fn feed_event<'a, 'b: 'a>( + &mut self, + op: Event<'a, 'b>, + _module_info: &ModuleInfo, + sink: &mut EventSink<'a, 'b>, + _source_loc: u32, + ) -> Result<(), Self::Error> { + match op { + Event::Internal(InternalEvent::FunctionBegin(_)) => { + self.current_block = self.func_locals_costs as u64; + } + Event::Wasm(&ref op) | Event::WasmOwned(ref op) => { + let opcode_index = get_opcode_index(op); + self.current_block += self.opcode_costs[opcode_index] as u64; + match *op { + Operator::Loop { .. } + | Operator::Block { .. } + | Operator::End + | Operator::If { .. } + | Operator::Else + | Operator::Unreachable + | Operator::Br { .. } + | Operator::BrTable { .. } + | Operator::BrIf { .. } + | Operator::Call { .. } + | Operator::CallIndirect { .. } + | Operator::Return => { + sink.push(Event::Internal(InternalEvent::GetInternal( + FIELD_USED_POINTS.index() as _, + ))); + sink.push(Event::WasmOwned(Operator::I64Const { + value: self.current_block as i64, + })); + sink.push(Event::WasmOwned(Operator::I64Add)); + sink.push(Event::Internal(InternalEvent::SetInternal( + FIELD_USED_POINTS.index() as _, + ))); + self.current_block = 0; + } + _ => {} + } + match *op { + Operator::Br { .. } + | Operator::BrTable { .. } + | Operator::BrIf { .. } + | Operator::Call { .. } + | Operator::CallIndirect { .. } => { + sink.push(Event::Internal(InternalEvent::GetInternal( + FIELD_USED_POINTS.index() as _, + ))); + sink.push(Event::Internal(InternalEvent::GetInternal( + FIELD_POINTS_LIMIT.index() as _, + ))); + sink.push(Event::WasmOwned(Operator::I64GeU)); + sink.push(Event::WasmOwned(Operator::If { + ty: WpTypeOrFuncType::Type(WpType::EmptyBlockType), + })); + push_runtime_breakpoint(sink, BREAKPOINT_VALUE_OUT_OF_GAS); + sink.push(Event::WasmOwned(Operator::End)); + } + _ => {} + } + } + _ => {} + } + + sink.push(op); + + Ok(()) + } + + fn feed_local( + &mut self, + _ty: WpType, + n: usize, + _loc: u32, + ) -> Result<(), Self::Error>{ + if n > self.unmetered_locals { + let metered_locals = (n - self.unmetered_locals) as u32; + let cost_index = get_local_allocate_cost_index(); + let cost = self.opcode_costs[cost_index]; + // n is already limited by Wasmparser; the following casting and multiplication are + // safe from overflowing + self.func_locals_costs += cost * metered_locals; + } + Ok(()) + } +} + +/// Returns the number of points used by an Instance. +pub fn get_points_used(instance: &Instance) -> u64 { + instance.get_internal(&FIELD_USED_POINTS) +} + +/// Sets the number of points used by an Instance. +pub fn set_points_used(instance: &mut Instance, value: u64) { + instance.set_internal(&FIELD_USED_POINTS, value); +} + +/// Sets the limit of points to be used by an Instance. +pub fn set_points_limit(instance: &mut Instance, value: u64) { + instance.set_internal(&FIELD_POINTS_LIMIT, value); +} + +/// Returns the number of points used in a Ctx. +pub fn get_points_used_ctx(ctx: &Ctx) -> u64 { + ctx.get_internal(&FIELD_USED_POINTS) +} + +/// Sets the number of points used in a Ctx. +pub fn set_points_used_ctx(ctx: &mut Ctx, value: u64) { + ctx.set_internal(&FIELD_USED_POINTS, value); +} diff --git a/lib/middleware-common/src/metering_costs.rs b/lib/middleware-common/src/metering_costs.rs new file mode 100644 index 000000000000..64c6f24646e6 --- /dev/null +++ b/lib/middleware-common/src/metering_costs.rs @@ -0,0 +1,457 @@ +use wasmer_runtime_core::wasmparser::Operator; + +pub fn get_local_allocate_cost_index() -> usize { + 447 +} + +pub fn get_opcode_index(op: &Operator) -> usize { + match *op { + Operator::Unreachable { .. } => { 0 } + Operator::Nop { .. } => { 1 } + Operator::Block { .. } => { 2 } + Operator::Loop { .. } => { 3 } + Operator::If { .. } => { 4 } + Operator::Else { .. } => { 5 } + Operator::End { .. } => { 6 } + Operator::Br { .. } => { 7 } + Operator::BrIf { .. } => { 8 } + Operator::BrTable { .. } => { 9 } + Operator::Return { .. } => { 10 } + Operator::Call { .. } => { 11 } + Operator::CallIndirect { .. } => { 12 } + Operator::Drop { .. } => { 13 } + Operator::Select { .. } => { 14 } + Operator::TypedSelect { .. } => { 15 } + Operator::LocalGet { .. } => { 16 } + Operator::LocalSet { .. } => { 17 } + Operator::LocalTee { .. } => { 18 } + Operator::GlobalGet { .. } => { 19 } + Operator::GlobalSet { .. } => { 20 } + Operator::I32Load { .. } => { 21 } + Operator::I64Load { .. } => { 22 } + Operator::F32Load { .. } => { 23 } + Operator::F64Load { .. } => { 24 } + Operator::I32Load8S { .. } => { 25 } + Operator::I32Load8U { .. } => { 26 } + Operator::I32Load16S { .. } => { 27 } + Operator::I32Load16U { .. } => { 28 } + Operator::I64Load8S { .. } => { 29 } + Operator::I64Load8U { .. } => { 30 } + Operator::I64Load16S { .. } => { 31 } + Operator::I64Load16U { .. } => { 32 } + Operator::I64Load32S { .. } => { 33 } + Operator::I64Load32U { .. } => { 34 } + Operator::I32Store { .. } => { 35 } + Operator::I64Store { .. } => { 36 } + Operator::F32Store { .. } => { 37 } + Operator::F64Store { .. } => { 38 } + Operator::I32Store8 { .. } => { 39 } + Operator::I32Store16 { .. } => { 40 } + Operator::I64Store8 { .. } => { 41 } + Operator::I64Store16 { .. } => { 42 } + Operator::I64Store32 { .. } => { 43 } + Operator::MemorySize { .. } => { 44 } + Operator::MemoryGrow { .. } => { 45 } + Operator::I32Const { .. } => { 46 } + Operator::I64Const { .. } => { 47 } + Operator::F32Const { .. } => { 48 } + Operator::F64Const { .. } => { 49 } + Operator::RefNull { .. } => { 50 } + Operator::RefIsNull { .. } => { 51 } + Operator::RefFunc { .. } => { 52 } + Operator::I32Eqz { .. } => { 53 } + Operator::I32Eq { .. } => { 54 } + Operator::I32Ne { .. } => { 55 } + Operator::I32LtS { .. } => { 56 } + Operator::I32LtU { .. } => { 57 } + Operator::I32GtS { .. } => { 58 } + Operator::I32GtU { .. } => { 59 } + Operator::I32LeS { .. } => { 60 } + Operator::I32LeU { .. } => { 61 } + Operator::I32GeS { .. } => { 62 } + Operator::I32GeU { .. } => { 63 } + Operator::I64Eqz { .. } => { 64 } + Operator::I64Eq { .. } => { 65 } + Operator::I64Ne { .. } => { 66 } + Operator::I64LtS { .. } => { 67 } + Operator::I64LtU { .. } => { 68 } + Operator::I64GtS { .. } => { 69 } + Operator::I64GtU { .. } => { 70 } + Operator::I64LeS { .. } => { 71 } + Operator::I64LeU { .. } => { 72 } + Operator::I64GeS { .. } => { 73 } + Operator::I64GeU { .. } => { 74 } + Operator::F32Eq { .. } => { 75 } + Operator::F32Ne { .. } => { 76 } + Operator::F32Lt { .. } => { 77 } + Operator::F32Gt { .. } => { 78 } + Operator::F32Le { .. } => { 79 } + Operator::F32Ge { .. } => { 80 } + Operator::F64Eq { .. } => { 81 } + Operator::F64Ne { .. } => { 82 } + Operator::F64Lt { .. } => { 83 } + Operator::F64Gt { .. } => { 84 } + Operator::F64Le { .. } => { 85 } + Operator::F64Ge { .. } => { 86 } + Operator::I32Clz { .. } => { 87 } + Operator::I32Ctz { .. } => { 88 } + Operator::I32Popcnt { .. } => { 89 } + Operator::I32Add { .. } => { 90 } + Operator::I32Sub { .. } => { 91 } + Operator::I32Mul { .. } => { 92 } + Operator::I32DivS { .. } => { 93 } + Operator::I32DivU { .. } => { 94 } + Operator::I32RemS { .. } => { 95 } + Operator::I32RemU { .. } => { 96 } + Operator::I32And { .. } => { 97 } + Operator::I32Or { .. } => { 98 } + Operator::I32Xor { .. } => { 99 } + Operator::I32Shl { .. } => { 100 } + Operator::I32ShrS { .. } => { 101 } + Operator::I32ShrU { .. } => { 102 } + Operator::I32Rotl { .. } => { 103 } + Operator::I32Rotr { .. } => { 104 } + Operator::I64Clz { .. } => { 105 } + Operator::I64Ctz { .. } => { 106 } + Operator::I64Popcnt { .. } => { 107 } + Operator::I64Add { .. } => { 108 } + Operator::I64Sub { .. } => { 109 } + Operator::I64Mul { .. } => { 110 } + Operator::I64DivS { .. } => { 111 } + Operator::I64DivU { .. } => { 112 } + Operator::I64RemS { .. } => { 113 } + Operator::I64RemU { .. } => { 114 } + Operator::I64And { .. } => { 115 } + Operator::I64Or { .. } => { 116 } + Operator::I64Xor { .. } => { 117 } + Operator::I64Shl { .. } => { 118 } + Operator::I64ShrS { .. } => { 119 } + Operator::I64ShrU { .. } => { 120 } + Operator::I64Rotl { .. } => { 121 } + Operator::I64Rotr { .. } => { 122 } + Operator::F32Abs { .. } => { 123 } + Operator::F32Neg { .. } => { 124 } + Operator::F32Ceil { .. } => { 125 } + Operator::F32Floor { .. } => { 126 } + Operator::F32Trunc { .. } => { 127 } + Operator::F32Nearest { .. } => { 128 } + Operator::F32Sqrt { .. } => { 129 } + Operator::F32Add { .. } => { 130 } + Operator::F32Sub { .. } => { 131 } + Operator::F32Mul { .. } => { 132 } + Operator::F32Div { .. } => { 133 } + Operator::F32Min { .. } => { 134 } + Operator::F32Max { .. } => { 135 } + Operator::F32Copysign { .. } => { 136 } + Operator::F64Abs { .. } => { 137 } + Operator::F64Neg { .. } => { 138 } + Operator::F64Ceil { .. } => { 139 } + Operator::F64Floor { .. } => { 140 } + Operator::F64Trunc { .. } => { 141 } + Operator::F64Nearest { .. } => { 142 } + Operator::F64Sqrt { .. } => { 143 } + Operator::F64Add { .. } => { 144 } + Operator::F64Sub { .. } => { 145 } + Operator::F64Mul { .. } => { 146 } + Operator::F64Div { .. } => { 147 } + Operator::F64Min { .. } => { 148 } + Operator::F64Max { .. } => { 149 } + Operator::F64Copysign { .. } => { 150 } + Operator::I32WrapI64 { .. } => { 151 } + Operator::I32TruncF32S { .. } => { 152 } + Operator::I32TruncF32U { .. } => { 153 } + Operator::I32TruncF64S { .. } => { 154 } + Operator::I32TruncF64U { .. } => { 155 } + Operator::I64ExtendI32S { .. } => { 156 } + Operator::I64ExtendI32U { .. } => { 157 } + Operator::I64TruncF32S { .. } => { 158 } + Operator::I64TruncF32U { .. } => { 159 } + Operator::I64TruncF64S { .. } => { 160 } + Operator::I64TruncF64U { .. } => { 161 } + Operator::F32ConvertI32S { .. } => { 162 } + Operator::F32ConvertI32U { .. } => { 163 } + Operator::F32ConvertI64S { .. } => { 164 } + Operator::F32ConvertI64U { .. } => { 165 } + Operator::F32DemoteF64 { .. } => { 166 } + Operator::F64ConvertI32S { .. } => { 167 } + Operator::F64ConvertI32U { .. } => { 168 } + Operator::F64ConvertI64S { .. } => { 169 } + Operator::F64ConvertI64U { .. } => { 170 } + Operator::F64PromoteF32 { .. } => { 171 } + Operator::I32ReinterpretF32 { .. } => { 172 } + Operator::I64ReinterpretF64 { .. } => { 173 } + Operator::F32ReinterpretI32 { .. } => { 174 } + Operator::F64ReinterpretI64 { .. } => { 175 } + Operator::I32Extend8S { .. } => { 176 } + Operator::I32Extend16S { .. } => { 177 } + Operator::I64Extend8S { .. } => { 178 } + Operator::I64Extend16S { .. } => { 179 } + Operator::I64Extend32S { .. } => { 180 } + Operator::I32TruncSatF32S { .. } => { 181 } + Operator::I32TruncSatF32U { .. } => { 182 } + Operator::I32TruncSatF64S { .. } => { 183 } + Operator::I32TruncSatF64U { .. } => { 184 } + Operator::I64TruncSatF32S { .. } => { 185 } + Operator::I64TruncSatF32U { .. } => { 186 } + Operator::I64TruncSatF64S { .. } => { 187 } + Operator::I64TruncSatF64U { .. } => { 188 } + Operator::MemoryInit { .. } => { 189 } + Operator::DataDrop { .. } => { 190 } + Operator::MemoryCopy { .. } => { 191 } + Operator::MemoryFill { .. } => { 192 } + Operator::TableInit { .. } => { 193 } + Operator::ElemDrop { .. } => { 194 } + Operator::TableCopy { .. } => { 195 } + Operator::TableFill { .. } => {194 } + Operator::TableGet { .. } => { 197 } + Operator::TableSet { .. } => { 198 } + Operator::TableGrow { .. } => { 199 } + Operator::TableSize { .. } => { 200 } + Operator::AtomicNotify { .. } => { 201 } + Operator::I32AtomicWait { .. } => { 202 } + Operator::I64AtomicWait { .. } => { 203 } + Operator::AtomicFence { .. } => { 204 } + Operator::I32AtomicLoad { .. } => { 205 } + Operator::I64AtomicLoad { .. } => { 206 } + Operator::I32AtomicLoad8U { .. } => { 207 } + Operator::I32AtomicLoad16U { .. } => { 208 } + Operator::I64AtomicLoad8U { .. } => { 209 } + Operator::I64AtomicLoad16U { .. } => { 210 } + Operator::I64AtomicLoad32U { .. } => { 211 } + Operator::I32AtomicStore { .. } => { 212 } + Operator::I64AtomicStore { .. } => { 213 } + Operator::I32AtomicStore8 { .. } => { 214 } + Operator::I32AtomicStore16 { .. } => { 215 } + Operator::I64AtomicStore8 { .. } => { 216 } + Operator::I64AtomicStore16 { .. } => { 217 } + Operator::I64AtomicStore32 { .. } => { 218 } + Operator::I32AtomicRmwAdd { .. } => { 219 } + Operator::I64AtomicRmwAdd { .. } => { 220 } + Operator::I32AtomicRmw8AddU { .. } => { 221 } + Operator::I32AtomicRmw16AddU { .. } => { 222 } + Operator::I64AtomicRmw8AddU { .. } => { 223 } + Operator::I64AtomicRmw16AddU { .. } => { 224 } + Operator::I64AtomicRmw32AddU { .. } => { 225 } + Operator::I32AtomicRmwSub { .. } => { 226 } + Operator::I64AtomicRmwSub { .. } => { 227 } + Operator::I32AtomicRmw8SubU { .. } => { 228 } + Operator::I32AtomicRmw16SubU { .. } => { 229 } + Operator::I64AtomicRmw8SubU { .. } => { 230 } + Operator::I64AtomicRmw16SubU { .. } => { 231 } + Operator::I64AtomicRmw32SubU { .. } => { 232 } + Operator::I32AtomicRmwAnd { .. } => { 233 } + Operator::I64AtomicRmwAnd { .. } => { 234 } + Operator::I32AtomicRmw8AndU { .. } => { 235 } + Operator::I32AtomicRmw16AndU { .. } => { 236 } + Operator::I64AtomicRmw8AndU { .. } => { 237 } + Operator::I64AtomicRmw16AndU { .. } => { 238 } + Operator::I64AtomicRmw32AndU { .. } => { 239 } + Operator::I32AtomicRmwOr { .. } => { 240 } + Operator::I64AtomicRmwOr { .. } => { 241 } + Operator::I32AtomicRmw8OrU { .. } => { 242 } + Operator::I32AtomicRmw16OrU { .. } => { 243 } + Operator::I64AtomicRmw8OrU { .. } => { 244 } + Operator::I64AtomicRmw16OrU { .. } => { 245 } + Operator::I64AtomicRmw32OrU { .. } => { 246 } + Operator::I32AtomicRmwXor { .. } => { 247 } + Operator::I64AtomicRmwXor { .. } => { 248 } + Operator::I32AtomicRmw8XorU { .. } => { 249 } + Operator::I32AtomicRmw16XorU { .. } => { 250 } + Operator::I64AtomicRmw8XorU { .. } => { 251 } + Operator::I64AtomicRmw16XorU { .. } => { 252 } + Operator::I64AtomicRmw32XorU { .. } => { 253 } + Operator::I32AtomicRmwXchg { .. } => { 254 } + Operator::I64AtomicRmwXchg { .. } => { 255 } + Operator::I32AtomicRmw8XchgU { .. } => { 256 } + Operator::I32AtomicRmw16XchgU { .. } => { 257 } + Operator::I64AtomicRmw8XchgU { .. } => { 258 } + Operator::I64AtomicRmw16XchgU { .. } => { 259 } + Operator::I64AtomicRmw32XchgU { .. } => { 260 } + Operator::I32AtomicRmwCmpxchg { .. } => { 261 } + Operator::I64AtomicRmwCmpxchg { .. } => { 262 } + Operator::I32AtomicRmw8CmpxchgU { .. } => { 263 } + Operator::I32AtomicRmw16CmpxchgU { .. } => { 264 } + Operator::I64AtomicRmw8CmpxchgU { .. } => { 265 } + Operator::I64AtomicRmw16CmpxchgU { .. } => { 266 } + Operator::I64AtomicRmw32CmpxchgU { .. } => { 267 } + Operator::V128Load { .. } => { 268 } + Operator::V128Store { .. } => { 269 } + Operator::V128Const { .. } => { 270 } + Operator::I8x16Splat { .. } => { 271 } + Operator::I8x16ExtractLaneS { .. } => { 272 } + Operator::I8x16ExtractLaneU { .. } => { 273 } + Operator::I8x16ReplaceLane { .. } => { 274 } + Operator::I16x8Splat { .. } => { 275 } + Operator::I16x8ExtractLaneS { .. } => { 276 } + Operator::I16x8ExtractLaneU { .. } => { 277 } + Operator::I16x8ReplaceLane { .. } => { 278 } + Operator::I32x4Splat { .. } => { 279 } + Operator::I32x4ExtractLane { .. } => { 280 } + Operator::I32x4ReplaceLane { .. } => { 281 } + Operator::I64x2Splat { .. } => { 282 } + Operator::I64x2ExtractLane { .. } => { 283 } + Operator::I64x2ReplaceLane { .. } => { 284 } + Operator::F32x4Splat { .. } => { 285 } + Operator::F32x4ExtractLane { .. } => { 286 } + Operator::F32x4ReplaceLane { .. } => { 287 } + Operator::F64x2Splat { .. } => { 288 } + Operator::F64x2ExtractLane { .. } => { 289 } + Operator::F64x2ReplaceLane { .. } => { 290 } + Operator::I8x16Eq { .. } => { 291 } + Operator::I8x16Ne { .. } => { 292 } + Operator::I8x16LtS { .. } => { 293 } + Operator::I8x16LtU { .. } => { 294 } + Operator::I8x16GtS { .. } => { 295 } + Operator::I8x16GtU { .. } => { 296 } + Operator::I8x16LeS { .. } => { 297 } + Operator::I8x16LeU { .. } => { 298 } + Operator::I8x16GeS { .. } => { 299 } + Operator::I8x16GeU { .. } => { 300 } + Operator::I16x8Eq { .. } => { 301 } + Operator::I16x8Ne { .. } => { 302 } + Operator::I16x8LtS { .. } => { 303 } + Operator::I16x8LtU { .. } => { 304 } + Operator::I16x8GtS { .. } => { 305 } + Operator::I16x8GtU { .. } => { 306 } + Operator::I16x8LeS { .. } => { 307 } + Operator::I16x8LeU { .. } => { 308 } + Operator::I16x8GeS { .. } => { 309 } + Operator::I16x8GeU { .. } => { 310 } + Operator::I32x4Eq { .. } => { 311 } + Operator::I32x4Ne { .. } => { 312 } + Operator::I32x4LtS { .. } => { 313 } + Operator::I32x4LtU { .. } => { 314 } + Operator::I32x4GtS { .. } => { 315 } + Operator::I32x4GtU { .. } => { 316 } + Operator::I32x4LeS { .. } => { 317 } + Operator::I32x4LeU { .. } => { 318 } + Operator::I32x4GeS { .. } => { 319 } + Operator::I32x4GeU { .. } => { 320 } + Operator::F32x4Eq { .. } => { 321 } + Operator::F32x4Ne { .. } => { 322 } + Operator::F32x4Lt { .. } => { 323 } + Operator::F32x4Gt { .. } => { 324 } + Operator::F32x4Le { .. } => { 325 } + Operator::F32x4Ge { .. } => { 326 } + Operator::F64x2Eq { .. } => { 327 } + Operator::F64x2Ne { .. } => { 328 } + Operator::F64x2Lt { .. } => { 329 } + Operator::F64x2Gt { .. } => { 330 } + Operator::F64x2Le { .. } => { 331 } + Operator::F64x2Ge { .. } => { 332 } + Operator::V128Not { .. } => { 333 } + Operator::V128And { .. } => { 334 } + Operator::V128AndNot { .. } => { 335 } + Operator::V128Or { .. } => { 336 } + Operator::V128Xor { .. } => { 337 } + Operator::V128Bitselect { .. } => { 338 } + Operator::I8x16Neg { .. } => { 339 } + Operator::I8x16AnyTrue { .. } => { 340 } + Operator::I8x16AllTrue { .. } => { 341 } + Operator::I8x16Shl { .. } => { 342 } + Operator::I8x16ShrS { .. } => { 343 } + Operator::I8x16ShrU { .. } => { 344 } + Operator::I8x16Add { .. } => { 345 } + Operator::I8x16AddSaturateS { .. } => { 346 } + Operator::I8x16AddSaturateU { .. } => { 347 } + Operator::I8x16Sub { .. } => { 348 } + Operator::I8x16SubSaturateS { .. } => { 349 } + Operator::I8x16SubSaturateU { .. } => { 350 } + Operator::I8x16MinS { .. } => { 354 }, + Operator::I8x16MinU { .. } => { 354 }, + Operator::I8x16MaxS { .. } => { 354 }, + Operator::I8x16MaxU { .. } => { 354 }, + Operator::I8x16Mul { .. } => { 355 } + Operator::I16x8Neg { .. } => { 356 } + Operator::I16x8AnyTrue { .. } => { 357 } + Operator::I16x8AllTrue { .. } => { 358 } + Operator::I16x8Shl { .. } => { 359 } + Operator::I16x8ShrS { .. } => { 360 } + Operator::I16x8ShrU { .. } => { 361 } + Operator::I16x8Add { .. } => { 362 } + Operator::I16x8AddSaturateS { .. } => { 363 } + Operator::I16x8AddSaturateU { .. } => { 364 } + Operator::I16x8Sub { .. } => { 365 } + Operator::I16x8SubSaturateS { .. } => { 366 } + Operator::I16x8SubSaturateU { .. } => { 367 } + Operator::I16x8Mul { .. } => { 368 } + Operator::I16x8MinS { .. } => { 369 } + Operator::I16x8MinU { .. } => { 370 } + Operator::I16x8MaxS { .. } => { 371 } + Operator::I16x8MaxU { .. } => { 372 } + Operator::I32x4Neg { .. } => { 373 } + Operator::I32x4AnyTrue { .. } => { 374 } + Operator::I32x4AllTrue { .. } => { 375 } + Operator::I32x4Shl { .. } => { 376 } + Operator::I32x4ShrS { .. } => { 377 } + Operator::I32x4ShrU { .. } => { 378 } + Operator::I32x4Add { .. } => { 379 } + Operator::I32x4Sub { .. } => { 380 } + Operator::I32x4Mul { .. } => { 381 } + Operator::I32x4MinS { .. } => { 382 } + Operator::I32x4MinU { .. } => { 383 } + Operator::I32x4MaxS { .. } => { 384 } + Operator::I32x4MaxU { .. } => { 385 } + Operator::I64x2Neg { .. } => { 386 } + Operator::I64x2AnyTrue { .. } => { 387 } + Operator::I64x2AllTrue { .. } => { 388 } + Operator::I64x2Shl { .. } => { 389 } + Operator::I64x2ShrS { .. } => { 390 } + Operator::I64x2ShrU { .. } => { 391 } + Operator::I64x2Add { .. } => { 392 } + Operator::I64x2Sub { .. } => { 393 } + Operator::I64x2Mul { .. } => { 394 } + Operator::F32x4Abs { .. } => { 395 } + Operator::F32x4Neg { .. } => { 396 } + Operator::F32x4Sqrt { .. } => { 397 } + Operator::F32x4Add { .. } => { 398 } + Operator::F32x4Sub { .. } => { 399 } + Operator::F32x4Mul { .. } => { 400 } + Operator::F32x4Div { .. } => { 401 } + Operator::F32x4Min { .. } => { 402 } + Operator::F32x4Max { .. } => { 403 } + Operator::F64x2Abs { .. } => { 404 } + Operator::F64x2Neg { .. } => { 405 } + Operator::F64x2Sqrt { .. } => { 406 } + Operator::F64x2Add { .. } => { 407 } + Operator::F64x2Sub { .. } => { 408 } + Operator::F64x2Mul { .. } => { 409 } + Operator::F64x2Div { .. } => { 410 } + Operator::F64x2Min { .. } => { 411 } + Operator::F64x2Max { .. } => { 412 } + Operator::I32x4TruncSatF32x4S { .. } => { 413 } + Operator::I32x4TruncSatF32x4U { .. } => { 414 } + Operator::I64x2TruncSatF64x2S { .. } => { 415 } + Operator::I64x2TruncSatF64x2U { .. } => { 416 } + Operator::F32x4ConvertI32x4S { .. } => { 417 } + Operator::F32x4ConvertI32x4U { .. } => { 418 } + Operator::F64x2ConvertI64x2S { .. } => { 419 } + Operator::F64x2ConvertI64x2U { .. } => { 420 } + Operator::V8x16Swizzle { .. } => { 421 } + Operator::V8x16Shuffle { .. } => { 422 } + Operator::V8x16LoadSplat { .. } => { 423 } + Operator::V16x8LoadSplat { .. } => { 424 } + Operator::V32x4LoadSplat { .. } => { 425 } + Operator::V64x2LoadSplat { .. } => { 426 } + Operator::I8x16NarrowI16x8S { .. } => { 427 } + Operator::I8x16NarrowI16x8U { .. } => { 428 } + Operator::I16x8NarrowI32x4S { .. } => { 429 } + Operator::I16x8NarrowI32x4U { .. } => { 430 } + Operator::I16x8WidenLowI8x16S { .. } => { 431 } + Operator::I16x8WidenHighI8x16S { .. } => { 432 } + Operator::I16x8WidenLowI8x16U { .. } => { 433 } + Operator::I16x8WidenHighI8x16U { .. } => { 434 } + Operator::I32x4WidenLowI16x8S { .. } => { 435 } + Operator::I32x4WidenHighI16x8S { .. } => { 436 } + Operator::I32x4WidenLowI16x8U { .. } => { 437 } + Operator::I32x4WidenHighI16x8U { .. } => { 438 } + Operator::I16x8Load8x8S { .. } => { 439 } + Operator::I16x8Load8x8U { .. } => { 440 } + Operator::I32x4Load16x4S { .. } => { 441 } + Operator::I32x4Load16x4U { .. } => { 442 } + Operator::I64x2Load32x2S { .. } => { 443 } + Operator::I64x2Load32x2U { .. } => { 444 } + Operator::I8x16RoundingAverageU { .. } => { 445 } + Operator::I16x8RoundingAverageU { .. } => { 446 } + } +} diff --git a/lib/middleware-common/src/opcode_control.rs b/lib/middleware-common/src/opcode_control.rs new file mode 100644 index 000000000000..8c85aec1a535 --- /dev/null +++ b/lib/middleware-common/src/opcode_control.rs @@ -0,0 +1,120 @@ +use wasmer_runtime_core::{ + codegen::{Event, EventSink, FunctionMiddleware, InternalEvent}, + wasmparser::{Operator, Type as WpType, TypeOrFuncType as WpTypeOrFuncType}, + vm::InternalField, + module::ModuleInfo, +}; + +use crate::runtime_breakpoints::{ + push_runtime_breakpoint, + BREAKPOINT_VALUE_MEMORY_LIMIT, +}; + +static FIELD_MEMORY_GROW_COUNT: InternalField = InternalField::allocate(); + +static FIELD_OPERAND_BACKUP: InternalField = InternalField::allocate(); + +pub struct OpcodeControl { + pub max_memory_grow: usize, + pub max_memory_grow_delta: usize, +} + +impl OpcodeControl { + pub fn new(max_memory_grow: usize, max_memory_grow_delta: usize) -> OpcodeControl { + OpcodeControl { + max_memory_grow, + max_memory_grow_delta, + } + } + + fn inject_memory_grow_count_limit(&mut self, sink: &mut EventSink) { + sink.push(Event::Internal(InternalEvent::GetInternal( + FIELD_MEMORY_GROW_COUNT.index() as _, + ))); + sink.push(Event::WasmOwned(Operator::I64Const { + value: self.max_memory_grow as i64, + })); + sink.push(Event::WasmOwned(Operator::I64GeU)); + sink.push(Event::WasmOwned(Operator::If { + ty: WpTypeOrFuncType::Type(WpType::EmptyBlockType), + })); + push_runtime_breakpoint(sink, BREAKPOINT_VALUE_MEMORY_LIMIT); + sink.push(Event::WasmOwned(Operator::End)); + } + + fn inject_memory_grow_count_increment(&mut self, sink: &mut EventSink) { + sink.push(Event::Internal(InternalEvent::GetInternal( + FIELD_MEMORY_GROW_COUNT.index() as _, + ))); + sink.push(Event::WasmOwned(Operator::I64Const{ + value: 1 as i64, + })); + sink.push(Event::WasmOwned(Operator::I64Add)); + sink.push(Event::Internal(InternalEvent::SetInternal( + FIELD_MEMORY_GROW_COUNT.index() as _, + ))); + } + + fn inject_memory_grow_delta_limit(&mut self, sink: &mut EventSink) { + sink.push(Event::Internal(InternalEvent::GetInternal( + FIELD_OPERAND_BACKUP.index() as _, + ))); + sink.push(Event::WasmOwned(Operator::I64Const { + value: self.max_memory_grow_delta as i64, + })); + sink.push(Event::WasmOwned(Operator::I64GtU)); + sink.push(Event::WasmOwned(Operator::If { + ty: WpTypeOrFuncType::Type(WpType::EmptyBlockType), + })); + push_runtime_breakpoint(sink, BREAKPOINT_VALUE_MEMORY_LIMIT); + sink.push(Event::WasmOwned(Operator::End)); + } +} + +impl FunctionMiddleware for OpcodeControl { + type Error = String; + fn feed_event<'a, 'b: 'a>( + &mut self, + op: Event<'a, 'b>, + _: &ModuleInfo, + sink: &mut EventSink<'a, 'b>, + _: u32, + ) -> Result<(), Self::Error> { + match op { + Event::Wasm(&ref op) | Event::WasmOwned(ref op) => { + match *op { + Operator::MemoryGrow { reserved } => { + if reserved != 0 { + return Err("MemoryGrow must have memory index 0".to_string()); + } + + // Before attempting anything with memory.grow, the current memory.grow + // count is checked against the self.max_memory_grow limit. + self.inject_memory_grow_count_limit(sink); + self.inject_memory_grow_count_increment(sink); + + // Backup the top of the stack (the parameter for memory.grow) in order to + // duplicate it: once for the comparison against max_memory_grow_delta and + // again for memory.grow itself, assuming the comparison passes. + sink.push(Event::Internal(InternalEvent::SetInternal( + FIELD_OPERAND_BACKUP.index() as _, + ))); + + // Set up the comparison against max_memory_grow_delta. + self.inject_memory_grow_delta_limit(sink); + + // Bring back the backed-up operand for memory.grow. + sink.push(Event::Internal(InternalEvent::GetInternal( + FIELD_OPERAND_BACKUP.index() as _, + ))); + } + _ => {} + } + } + _ => {} + } + + sink.push(op); + Ok(()) + } +} diff --git a/lib/middleware-common/src/opcode_trace.rs b/lib/middleware-common/src/opcode_trace.rs new file mode 100644 index 000000000000..57920e14b9c9 --- /dev/null +++ b/lib/middleware-common/src/opcode_trace.rs @@ -0,0 +1,95 @@ +use std::fs::File; +use std::io; +use std::io::Write; + +use wasmer_runtime_core::{ + codegen::{Event, EventSink, FunctionMiddleware, InternalEvent}, + module::ModuleInfo, + vm::InternalField, + wasmparser::Operator, + Instance, +}; + + +static OPCODE_LAST_LOCATION: InternalField = InternalField::allocate(); + +pub struct OpcodeTracer { + pub output_file: File, +} + +impl OpcodeTracer { + pub fn new() -> OpcodeTracer { + OpcodeTracer { + output_file: File::create("opcode.trace").unwrap(), + } + } + + pub fn trace_instance_exports(&mut self, instance: &Instance) -> io::Result<()> { + write!(self.output_file, "{:#?}\n", instance.module.info.exports) + } + + pub fn trace_event(&mut self, ev: &Event, source_loc: u32) -> io::Result<()> { + match *ev { + Event::Internal(InternalEvent::FunctionBegin(function_index)) => { + self.trace_function_begin(function_index) + } + Event::Internal(InternalEvent::FunctionEnd) => { + self.trace_function_end() + } + _ => { + self.trace_opcode_event(ev, source_loc) + } + } + } + + pub fn trace_function_begin(&mut self, function_index: u32) -> io::Result<()>{ + write!(self.output_file, "FUNCTION BEGIN: {}\n", function_index) + } + + pub fn trace_function_end(&mut self) -> io::Result<()> { + write!(self.output_file, "FUNCTION END\n") + } + + pub fn trace_opcode_event(&mut self, ev: &Event, source_loc: u32) -> io::Result<()> { + match ev { + Event::Wasm(&ref op) + | Event::WasmOwned(ref op) => { + write!(self.output_file, "\t{}:\t{:?}\n", source_loc, *op) + } + _ => Ok(()) + } + } + + pub fn push_last_location_tracer(&self, sink: &mut EventSink, source_loc: u32) { + sink.push(Event::WasmOwned(Operator::I64Const { + value: source_loc as i64, + })); + sink.push(Event::Internal(InternalEvent::SetInternal( + OPCODE_LAST_LOCATION.index() as _, + ))); + } +} + +impl FunctionMiddleware for OpcodeTracer { + type Error = String; + fn feed_event<'a, 'b: 'a>( + &mut self, + op: Event<'a, 'b>, + _module_info: &ModuleInfo, + sink: &mut EventSink<'a, 'b>, + source_loc: u32, + ) -> Result<(), Self::Error> { + self.trace_event(&op, source_loc).expect("failed to trace event"); + self.push_last_location_tracer(sink, source_loc); + sink.push(op); + Ok(()) + } +} + +pub fn get_opcodetracer_last_location(instance: &mut Instance) -> u64 { + instance.get_internal(&OPCODE_LAST_LOCATION) +} + +pub fn reset_opcodetracer_last_location(instance: &mut Instance) { + instance.set_internal(&OPCODE_LAST_LOCATION, 0); +} diff --git a/lib/middleware-common/src/runtime_breakpoints.rs b/lib/middleware-common/src/runtime_breakpoints.rs new file mode 100644 index 000000000000..773d91594477 --- /dev/null +++ b/lib/middleware-common/src/runtime_breakpoints.rs @@ -0,0 +1,89 @@ +use wasmer_runtime_core::{ + codegen::{Event, EventSink, FunctionMiddleware, InternalEvent}, + module::ModuleInfo, + vm::InternalField, + wasmparser::{Operator, Type as WpType, TypeOrFuncType as WpTypeOrFuncType}, + error::RuntimeError, + Instance, +}; + +pub static FIELD_RUNTIME_BREAKPOINT_VALUE: InternalField = InternalField::allocate(); +pub const BREAKPOINT_VALUE_NO_BREAKPOINT: u64 = 0; +pub const BREAKPOINT_VALUE_EXECUTION_FAILED: u64 = 1; +pub const BREAKPOINT_VALUE_OUT_OF_GAS: u64 = 4; +pub const BREAKPOINT_VALUE_MEMORY_LIMIT: u64 = 5; + + +pub struct RuntimeBreakpointHandler {} + +impl RuntimeBreakpointHandler { + pub fn new() -> RuntimeBreakpointHandler { + RuntimeBreakpointHandler {} + } +} + +impl FunctionMiddleware for RuntimeBreakpointHandler { + type Error = String; + fn feed_event<'a, 'b: 'a>( + &mut self, + op: Event<'a, 'b>, + _module_info: &ModuleInfo, + sink: &mut EventSink<'a, 'b>, + _source_loc: u32, + ) -> Result<(), Self::Error> { + + let must_add_breakpoint = match op { + Event::Wasm(&ref op) | Event::WasmOwned(ref op) => { + match *op { + Operator::Call { .. } + | Operator::CallIndirect { .. } => { + true + } + _ => false + } + } + _ => false + }; + + sink.push(op); + + if must_add_breakpoint { + sink.push(Event::Internal(InternalEvent::GetInternal( + FIELD_RUNTIME_BREAKPOINT_VALUE.index() as _, + ))); + sink.push(Event::WasmOwned(Operator::I64Const { + value: BREAKPOINT_VALUE_NO_BREAKPOINT as i64, + })); + sink.push(Event::WasmOwned(Operator::I64Ne)); + sink.push(Event::WasmOwned(Operator::If { + ty: WpTypeOrFuncType::Type(WpType::EmptyBlockType), + })); + sink.push(Event::Internal(InternalEvent::Breakpoint(Box::new(|_| { + Err(Box::new(RuntimeError(Box::new("breakpoint reached".to_string())))) + })))); + sink.push(Event::WasmOwned(Operator::End)); + } + + Ok(()) + } +} + +pub fn push_runtime_breakpoint(sink: &mut EventSink, value: u64) { + sink.push(Event::WasmOwned(Operator::I64Const { + value: value as i64, + })); + sink.push(Event::Internal(InternalEvent::SetInternal( + FIELD_RUNTIME_BREAKPOINT_VALUE.index() as _, + ))); + sink.push(Event::Internal(InternalEvent::Breakpoint(Box::new(|_| { + Err(Box::new(RuntimeError(Box::new("breakpoint reached".to_string())))) + })))); +} + +pub fn set_runtime_breakpoint_value(instance: &mut Instance, value: u64) { + instance.set_internal(&FIELD_RUNTIME_BREAKPOINT_VALUE, value); +} + +pub fn get_runtime_breakpoint_value(instance: &mut Instance) -> u64 { + instance.get_internal(&FIELD_RUNTIME_BREAKPOINT_VALUE) +} diff --git a/lib/runtime-c-api/Cargo.toml b/lib/runtime-c-api/Cargo.toml new file mode 100644 index 000000000000..efce4f34e0aa --- /dev/null +++ b/lib/runtime-c-api/Cargo.toml @@ -0,0 +1,77 @@ +[package] +name = "wasmer-runtime-c-api" +version = "0.15.0" +description = "Wasmer C API library" +documentation = "https://wasmerio.github.io/wasmer/c/runtime-c-api/" +license = "MIT" +authors = ["The Wasmer Engineering Team "] +repository = "https://github.com/ElrondNetwork/wasmer" +keywords = ["wasm", "webassembly", "runtime"] +categories = ["wasm"] +edition = "2018" +readme = "README.md" + +[lib] +crate-type = ["cdylib", "rlib", "staticlib"] + +[dependencies] +libc = "0.2.60" + +[dependencies.rkyv] +version = "0.7.26" +features = ["indexmap"] + +[dependencies.wasmer-runtime] +default-features = false +path = "../runtime" +version = "0.15.0" + +[dependencies.wasmer-runtime-core] +default-features = false +path = "../runtime-core" +version = "0.15.0" + +[dependencies.wasmer-wasi] +default-features = false +path = "../wasi" +version = "0.15.0" +optional = true + +[dependencies.wasmer-emscripten] +path = "../emscripten" +version = "0.15.0" +optional = true + +[dependencies.wasmer-middleware-common] +path = "../middleware-common" +version = "0.15.0" +optional = true + +[dependencies.wasmer-singlepass-backend] +path = "../singlepass-backend" +version = "0.15.0" +optional = true + +[dependencies.wasmer-llvm-backend] +path = "../llvm-backend" +version = "0.15.0" +optional = true + +[dependencies.wasmer-clif-backend] +path = "../clif-backend" +version = "0.15.0" +optional = true + +[features] +default = ["singlepass-backend", "metering", "runtime-breakpoints", "wasmer-runtime/deterministic-execution"] +debug = ["wasmer-runtime/debug"] +cranelift-backend = ["wasmer-runtime/cranelift", "wasmer-runtime/default-backend-cranelift"] +llvm-backend = ["wasmer-runtime/llvm", "wasmer-runtime/default-backend-llvm", "wasmer-llvm-backend"] +singlepass-backend = ["wasmer-runtime/singlepass", "wasmer-runtime/default-backend-singlepass", "wasmer-singlepass-backend", "wasmer-singlepass-backend/deterministic-execution"] +metering = ["wasmer-middleware-common"] +runtime-breakpoints = ["wasmer-middleware-common"] +wasi = ["wasmer-wasi"] +emscripten = ["wasmer-emscripten"] + +[build-dependencies] +cbindgen = "0.9" diff --git a/lib/runtime-c-api/README.md b/lib/runtime-c-api/README.md new file mode 100644 index 000000000000..4520e5a3bb7b --- /dev/null +++ b/lib/runtime-c-api/README.md @@ -0,0 +1,165 @@ +

+ + Wasmer logo + +

+ +

+ + Build Status + + + License + + + Join the Wasmer Community + + + Number of downloads from crates.io + + + Wasmer C API Documentation + +

+ +# Wasmer Runtime C API + +Wasmer is a standalone JIT WebAssembly runtime, aiming to be fully +compatible with WASI, Emscripten, Rust and Go. [Learn +more](https://github.com/wasmerio/wasmer). + +This crate exposes a C and a C++ API for the Wasmer runtime. + +# Usage + +The C and C++ header files can be found in the source tree of this +crate, respectively [`wasmer.h`][wasmer_h] and +[`wasmer.hh`][wasmer_hh]. They are automatically generated, and always +up-to-date in this repository. +The runtime shared library (so, dll, dylib) can also be downloaded in Wasmer [release page](https://github.com/wasmerio/wasmer/releases). + +You can find the full C API documentation here: +https://wasmerio.github.io/wasmer/c/runtime-c-api/ + +Here is a simple example to use the C API: + +```c +#include +#include "../wasmer.h" +#include +#include + +int main() +{ + // Read the Wasm file bytes. + FILE *file = fopen("sum.wasm", "r"); + fseek(file, 0, SEEK_END); + long len = ftell(file); + uint8_t *bytes = malloc(len); + fseek(file, 0, SEEK_SET); + fread(bytes, 1, len, file); + fclose(file); + + // Prepare the imports. + wasmer_import_t imports[] = {}; + + // Instantiate! + wasmer_instance_t *instance = NULL; + wasmer_result_t instantiation_result = wasmer_instantiate(&instance, bytes, len, imports, 0); + + assert(instantiation_result == WASMER_OK); + + // Let's call a function. + // Start by preparing the arguments. + + // Value of argument #1 is `7i32`. + wasmer_value_t argument_one; + argument_one.tag = WASM_I32; + argument_one.value.I32 = 7; + + // Value of argument #2 is `8i32`. + wasmer_value_t argument_two; + argument_two.tag = WASM_I32; + argument_two.value.I32 = 8; + + // Prepare the arguments. + wasmer_value_t arguments[] = {argument_one, argument_two}; + + // Prepare the return value. + wasmer_value_t result_one; + wasmer_value_t results[] = {result_one}; + + // Call the `sum` function with the prepared arguments and the return value. + wasmer_result_t call_result = wasmer_instance_call(instance, "sum", arguments, 2, results, 1); + + // Let's display the result. + printf("Call result: %d\n", call_result); + printf("Result: %d\n", results[0].value.I32); + + // `sum(7, 8) == 15`. + assert(results[0].value.I32 == 15); + assert(call_result == WASMER_OK); + + wasmer_instance_destroy(instance); + + return 0; +} +``` + +# Testing + +Tests are run using the release build of the library. If you make +changes or compile with non-default features, please ensure you +rebuild in release mode for the tests to see the changes. + +The tests can be run via `cargo test`, such as: + +```sh +$ cargo test --release -- --nocapture +``` + +To run tests manually, enter the `lib/runtime-c-api/tests` directory +and run the following commands: + +```sh +$ cmake . +$ make +$ make test +``` + +# Feature Flags + +By default this uses the cranelift compiler. You can enable a different compiler +by using feature flags. However, make sure to disable default features, as exactly +one default compiler must be enabled. + +LLVM: `cargo build --no-default-features --features llvm-backend` +single-pass: `cargo build --no-default-features --features singlepass-backend` + +Note that the single-pass backend +[does not currently support serialization](https://github.com/wasmerio/wasmer/issues/811), +so the serialization related tests will fail there. + +There is also a flag to enable gas metering which provides three new api endpoints: + +These are replaced with stubs of a normal compiler if the metering flag is not provided, +to provide a consistent API and not break upstream packages at link time. + +Note that the gas metering middleware is +[not currently compatible with the cranelift backend](https://github.com/wasmerio/wasmer/issues/819), +so you must select a different backend (see above) if you enable metering.eg + +`cargo build --no-default-features --features llvm-backend,metering` + +Once that issue is resolved, `cargo build --features metering` will give you a useful binary. + +# License + +Wasmer is primarily distributed under the terms of the [MIT +license][mit-license] ([LICENSE][license]). + + +[wasmer_h]: ./wasmer.h +[wasmer_hh]: ./wasmer.hh +[mit-license]: http://opensource.org/licenses/MIT +[license]: https://github.com/wasmerio/wasmer/blob/master/LICENSE diff --git a/lib/runtime-c-api/src/import/mod.rs b/lib/runtime-c-api/src/import/mod.rs new file mode 100644 index 000000000000..ad535680dc00 --- /dev/null +++ b/lib/runtime-c-api/src/import/mod.rs @@ -0,0 +1,938 @@ +//! Create, read, destroy import definitions (function, global, memory +//! and table) on an instance. + +use crate::{ + error::{update_last_error, CApiError}, + export::{wasmer_import_export_kind, wasmer_import_export_value}, + instance::wasmer_instance_context_t, + module::wasmer_module_t, + value::wasmer_value_tag, + wasmer_byte_array, wasmer_result_t, +}; +use libc::c_uint; +use std::{ + convert::TryFrom, + ffi::{c_void, CStr}, + os::raw::c_char, + ptr, slice, + sync::Arc, + result::Result, +}; +use wasmer_runtime::{Ctx, Global, Memory, Module, Table}; +use wasmer_runtime_core::{ + export::{Context, Export, FuncPointer}, + import::{ImportObject, Namespace, ImportObjectIterator}, + module::ImportName, + types::{FuncSig, Type}, +}; +use std::{collections::HashMap}; + +pub enum ImportError { + ModuleNameError, + ImportNameError, +} + +pub static mut GLOBAL_IMPORT_OBJECT: *mut ImportObject = 0 as *mut ImportObject; + +#[repr(C)] +pub struct wasmer_import_t { + pub module_name: wasmer_byte_array, + pub import_name: wasmer_byte_array, + pub tag: wasmer_import_export_kind, + pub value: wasmer_import_export_value, +} + +#[repr(C)] +pub struct wasmer_import_object_t; + +#[repr(C)] +#[derive(Clone)] +pub struct wasmer_import_func_t; + +#[repr(C)] +#[derive(Clone)] +pub struct wasmer_import_descriptor_t; + +#[repr(C)] +#[derive(Clone)] +pub struct wasmer_import_descriptors_t; + +#[repr(C)] +#[derive(Clone)] +pub struct wasmer_import_object_iter_t; + +/// Creates a new empty import object. +/// See also `wasmer_import_object_append` +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub unsafe extern "C" fn wasmer_import_object_new() -> *mut wasmer_import_object_t { + let import_object = Box::new(ImportObject::new()); + + Box::into_raw(import_object) as *mut wasmer_import_object_t +} + +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub unsafe extern "C" fn wasmer_import_object_cache_from_imports( + imports: *mut wasmer_import_t, + imports_len: c_uint, +) -> wasmer_result_t { + let imports_result = wasmer_create_import_object_from_imports(imports, imports_len); + let import_object = match imports_result { + Err(ImportError::ModuleNameError) => { + update_last_error(CApiError { msg: "error converting module name to string".to_string() }); + return wasmer_result_t::WASMER_ERROR; + } + Err(ImportError::ImportNameError) => { + update_last_error(CApiError { msg: "error converting import_name to string".to_string() }); + return wasmer_result_t::WASMER_ERROR; + } + Ok(created_imports_object) => created_imports_object + }; + + if GLOBAL_IMPORT_OBJECT != (0 as *mut ImportObject) { + let _ = Box::from_raw(GLOBAL_IMPORT_OBJECT); // deallocate previous GLOBAL_IMPORT_OBJECT + } + + GLOBAL_IMPORT_OBJECT = Box::into_raw(Box::new(import_object)); + return wasmer_result_t::WASMER_OK +} + +/// Assembles an ImportObject from a list of imports received on the C API +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub unsafe fn wasmer_create_import_object_from_imports( + imports: *mut wasmer_import_t, + imports_len: c_uint, +) -> Result { + let imports: &[wasmer_import_t] = slice::from_raw_parts(imports, imports_len as usize); + let mut import_object = ImportObject::new(); + let mut namespaces = HashMap::new(); + + for import in imports { + let module_name = slice::from_raw_parts( + import.module_name.bytes, + import.module_name.bytes_len as usize, + ); + let module_name = if let Ok(s) = std::str::from_utf8(module_name) { + s + } else { + return Err(ImportError::ModuleNameError) + }; + let import_name = slice::from_raw_parts( + import.import_name.bytes, + import.import_name.bytes_len as usize, + ); + let import_name = if let Ok(s) = std::str::from_utf8(import_name) { + s + } else { + return Err(ImportError::ImportNameError) + }; + + let namespace = namespaces.entry(module_name).or_insert_with(Namespace::new); + + let export = match import.tag { + wasmer_import_export_kind::WASM_MEMORY => { + let mem = import.value.memory as *mut Memory; + Export::Memory((&*mem).clone()) + } + wasmer_import_export_kind::WASM_FUNCTION => { + let func_export = import.value.func as *mut Export; + (&*func_export).clone() + } + wasmer_import_export_kind::WASM_GLOBAL => { + let global = import.value.global as *mut Global; + Export::Global((&*global).clone()) + } + wasmer_import_export_kind::WASM_TABLE => { + let table = import.value.table as *mut Table; + Export::Table((&*table).clone()) + } + }; + namespace.insert(import_name, export); + } + + for (module_name, namespace) in namespaces.into_iter() { + import_object.register(module_name, namespace); + } + + Ok(import_object) +} + +#[cfg(feature = "wasi")] +mod wasi; + +#[cfg(feature = "wasi")] +pub use self::wasi::*; + +#[cfg(feature = "emscripten")] +mod emscripten; + +#[cfg(feature = "emscripten")] +pub use self::emscripten::*; + +/// Gets an entry from an ImportObject at the name and namespace. +/// Stores `name`, `namespace`, and `import_export_value` in `import`. +/// Thus these must remain valid for the lifetime of `import`. +/// +/// The caller owns all data involved. +/// `import_export_value` will be written to based on `tag`. +#[no_mangle] +pub unsafe extern "C" fn wasmer_import_object_get_import( + import_object: *const wasmer_import_object_t, + namespace: wasmer_byte_array, + name: wasmer_byte_array, + import: *mut wasmer_import_t, + import_export_value: *mut wasmer_import_export_value, + tag: u32, +) -> wasmer_result_t { + let tag: wasmer_import_export_kind = if let Ok(t) = TryFrom::try_from(tag) { + t + } else { + update_last_error(CApiError { + msg: "wasmer_import_export_tag out of range".to_string(), + }); + return wasmer_result_t::WASMER_ERROR; + }; + let import_object: &mut ImportObject = &mut *(import_object as *mut ImportObject); + let namespace_str = if let Ok(ns) = namespace.as_str() { + ns + } else { + update_last_error(CApiError { + msg: "error converting namespace to UTF-8 string".to_string(), + }); + return wasmer_result_t::WASMER_ERROR; + }; + let name_str = if let Ok(name) = name.as_str() { + name + } else { + update_last_error(CApiError { + msg: "error converting name to UTF-8 string".to_string(), + }); + return wasmer_result_t::WASMER_ERROR; + }; + if import.is_null() || import_export_value.is_null() { + update_last_error(CApiError { + msg: "pointers to import and import_export_value must not be null".to_string(), + }); + return wasmer_result_t::WASMER_ERROR; + } + let import_out = &mut *import; + let import_export_value_out = &mut *import_export_value; + if let Some(export) = + import_object.maybe_with_namespace(namespace_str, |ns| ns.get_export(name_str)) + { + match export { + Export::Function { .. } => { + if tag != wasmer_import_export_kind::WASM_FUNCTION { + update_last_error(CApiError { + msg: format!("Found function, expected {}", tag.to_str()), + }); + return wasmer_result_t::WASMER_ERROR; + } + import_out.tag = wasmer_import_export_kind::WASM_FUNCTION; + let writer = import_export_value_out.func as *mut Export; + *writer = export.clone(); + } + Export::Memory(memory) => { + if tag != wasmer_import_export_kind::WASM_MEMORY { + update_last_error(CApiError { + msg: format!("Found memory, expected {}", tag.to_str()), + }); + return wasmer_result_t::WASMER_ERROR; + } + import_out.tag = wasmer_import_export_kind::WASM_MEMORY; + let writer = import_export_value_out.func as *mut Memory; + *writer = memory.clone(); + } + Export::Table(table) => { + if tag != wasmer_import_export_kind::WASM_TABLE { + update_last_error(CApiError { + msg: format!("Found table, expected {}", tag.to_str()), + }); + return wasmer_result_t::WASMER_ERROR; + } + import_out.tag = wasmer_import_export_kind::WASM_TABLE; + let writer = import_export_value_out.func as *mut Table; + *writer = table.clone(); + } + Export::Global(global) => { + if tag != wasmer_import_export_kind::WASM_GLOBAL { + update_last_error(CApiError { + msg: format!("Found global, expected {}", tag.to_str()), + }); + return wasmer_result_t::WASMER_ERROR; + } + import_out.tag = wasmer_import_export_kind::WASM_GLOBAL; + let writer = import_export_value_out.func as *mut Global; + *writer = global.clone(); + } + } + + import_out.value = *import_export_value; + import_out.module_name = namespace; + import_out.import_name = name; + + wasmer_result_t::WASMER_OK + } else { + update_last_error(CApiError { + msg: format!("Export {} {} not found", namespace_str, name_str), + }); + wasmer_result_t::WASMER_ERROR + } +} + +/// private wrapper data type used for casting +#[repr(C)] +struct WasmerImportObjectIterator( + std::iter::Peekable::Item>>>, +); + +/// Create an iterator over the functions in the import object. +/// Get the next import with `wasmer_import_object_iter_next` +/// Free the iterator with `wasmer_import_object_iter_destroy` +#[no_mangle] +pub unsafe extern "C" fn wasmer_import_object_iterate_functions( + import_object: *const wasmer_import_object_t, +) -> *mut wasmer_import_object_iter_t { + if import_object.is_null() { + update_last_error(CApiError { + msg: "import_object must not be null".to_owned(), + }); + return std::ptr::null_mut(); + } + let import_object: &ImportObject = &*(import_object as *const ImportObject); + let iter_inner = Box::new(import_object.clone_ref().into_iter().filter(|(_, _, e)| { + if let Export::Function { .. } = e { + true + } else { + false + } + })) as Box::Item>>; + let iterator = Box::new(WasmerImportObjectIterator(iter_inner.peekable())); + + Box::into_raw(iterator) as *mut wasmer_import_object_iter_t +} + +/// Writes the next value to `import`. `WASMER_ERROR` is returned if there +/// was an error or there's nothing left to return. +/// +/// To free the memory allocated here, pass the import to `wasmer_import_object_imports_destroy`. +/// To check if the iterator is done, use `wasmer_import_object_iter_at_end`. +#[no_mangle] +pub unsafe extern "C" fn wasmer_import_object_iter_next( + import_object_iter: *mut wasmer_import_object_iter_t, + import: *mut wasmer_import_t, +) -> wasmer_result_t { + if import_object_iter.is_null() || import.is_null() { + update_last_error(CApiError { + msg: "import_object_iter and import must not be null".to_owned(), + }); + return wasmer_result_t::WASMER_ERROR; + } + + let iter = &mut *(import_object_iter as *mut WasmerImportObjectIterator); + let out = &mut *import; + // TODO: the copying here can be optimized away, we just need to use a different type of + // iterator internally + if let Some((namespace, name, export)) = iter.0.next() { + let ns = { + let mut n = namespace.clone(); + n.shrink_to_fit(); + n.into_bytes() + }; + let ns_bytes = wasmer_byte_array { + bytes: ns.as_ptr(), + bytes_len: ns.len() as u32, + }; + + let name = { + let mut n = name.clone(); + n.shrink_to_fit(); + n.into_bytes() + }; + let name_bytes = wasmer_byte_array { + bytes: name.as_ptr(), + bytes_len: name.len() as u32, + }; + + out.module_name = ns_bytes; + out.import_name = name_bytes; + + std::mem::forget(ns); + std::mem::forget(name); + + match export { + Export::Function { .. } => { + let func = Box::new(export.clone()); + + out.tag = wasmer_import_export_kind::WASM_FUNCTION; + out.value = wasmer_import_export_value { + func: Box::into_raw(func) as *mut _ as *const _, + }; + } + Export::Global(global) => { + let glbl = Box::new(global.clone()); + + out.tag = wasmer_import_export_kind::WASM_GLOBAL; + out.value = wasmer_import_export_value { + global: Box::into_raw(glbl) as *mut _ as *const _, + }; + } + Export::Memory(memory) => { + let mem = Box::new(memory.clone()); + + out.tag = wasmer_import_export_kind::WASM_MEMORY; + out.value = wasmer_import_export_value { + memory: Box::into_raw(mem) as *mut _ as *const _, + }; + } + Export::Table(table) => { + let tbl = Box::new(table.clone()); + + out.tag = wasmer_import_export_kind::WASM_TABLE; + out.value = wasmer_import_export_value { + memory: Box::into_raw(tbl) as *mut _ as *const _, + }; + } + } + + wasmer_result_t::WASMER_OK + } else { + wasmer_result_t::WASMER_ERROR + } +} + +/// Returns true if further calls to `wasmer_import_object_iter_next` will +/// not return any new data +#[no_mangle] +pub unsafe extern "C" fn wasmer_import_object_iter_at_end( + import_object_iter: *mut wasmer_import_object_iter_t, +) -> bool { + if import_object_iter.is_null() { + update_last_error(CApiError { + msg: "import_object_iter must not be null".to_owned(), + }); + return true; + } + let iter = &mut *(import_object_iter as *mut WasmerImportObjectIterator); + + iter.0.peek().is_none() +} + +/// Frees the memory allocated by `wasmer_import_object_iterate_functions` +#[no_mangle] +pub unsafe extern "C" fn wasmer_import_object_iter_destroy( + import_object_iter: *mut wasmer_import_object_iter_t, +) { + if !import_object_iter.is_null() { + let _ = Box::from_raw(import_object_iter as *mut WasmerImportObjectIterator); + } +} + +/// Frees the memory allocated in `wasmer_import_object_iter_next` +/// +/// This function does not free the memory in `wasmer_import_object_t`; +/// it only frees memory allocated while querying a `wasmer_import_object_t`. +#[no_mangle] +pub unsafe extern "C" fn wasmer_import_object_imports_destroy( + imports: *mut wasmer_import_t, + imports_len: u32, +) { + if imports.is_null() { + return; + } + let imports: &[wasmer_import_t] = &*slice::from_raw_parts_mut(imports, imports_len as usize); + for import in imports { + let _namespace: Vec = Vec::from_raw_parts( + import.module_name.bytes as *mut u8, + import.module_name.bytes_len as usize, + import.module_name.bytes_len as usize, + ); + let _name: Vec = Vec::from_raw_parts( + import.import_name.bytes as *mut u8, + import.import_name.bytes_len as usize, + import.import_name.bytes_len as usize, + ); + match import.tag { + wasmer_import_export_kind::WASM_FUNCTION => { + let _: Box = Box::from_raw(import.value.func as *mut _); + } + wasmer_import_export_kind::WASM_GLOBAL => { + let _: Box = Box::from_raw(import.value.global as *mut _); + } + wasmer_import_export_kind::WASM_MEMORY => { + let _: Box = Box::from_raw(import.value.memory as *mut _); + } + wasmer_import_export_kind::WASM_TABLE => { + let _: Box = Box::from_raw(import.value.table as *mut _); + } + } + } +} + +/// Extends an existing import object with new imports +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub unsafe extern "C" fn wasmer_import_object_extend( + import_object: *mut wasmer_import_object_t, + imports: *const wasmer_import_t, + imports_len: c_uint, +) -> wasmer_result_t { + let import_object: &mut ImportObject = &mut *(import_object as *mut ImportObject); + + let mut extensions: Vec<(String, String, Export)> = Vec::new(); + + let imports: &[wasmer_import_t] = slice::from_raw_parts(imports, imports_len as usize); + for import in imports { + let module_name = slice::from_raw_parts( + import.module_name.bytes, + import.module_name.bytes_len as usize, + ); + let module_name = if let Ok(s) = std::str::from_utf8(module_name) { + s + } else { + update_last_error(CApiError { + msg: "error converting module name to string".to_string(), + }); + return wasmer_result_t::WASMER_ERROR; + }; + let import_name = slice::from_raw_parts( + import.import_name.bytes, + import.import_name.bytes_len as usize, + ); + let import_name = if let Ok(s) = std::str::from_utf8(import_name) { + s + } else { + update_last_error(CApiError { + msg: "error converting import_name to string".to_string(), + }); + return wasmer_result_t::WASMER_ERROR; + }; + + let export = match import.tag { + wasmer_import_export_kind::WASM_MEMORY => { + let mem = import.value.memory as *mut Memory; + Export::Memory((&*mem).clone()) + } + wasmer_import_export_kind::WASM_FUNCTION => { + let func_export = import.value.func as *mut Export; + (&*func_export).clone() + } + wasmer_import_export_kind::WASM_GLOBAL => { + let global = import.value.global as *mut Global; + Export::Global((&*global).clone()) + } + wasmer_import_export_kind::WASM_TABLE => { + let table = import.value.table as *mut Table; + Export::Table((&*table).clone()) + } + }; + + let extension = (module_name.to_string(), import_name.to_string(), export); + extensions.push(extension) + } + + import_object.extend(extensions); + + return wasmer_result_t::WASMER_OK; +} + +/// Gets import descriptors for the given module +/// +/// The caller owns the object and should call `wasmer_import_descriptors_destroy` to free it. +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub unsafe extern "C" fn wasmer_import_descriptors( + module: *const wasmer_module_t, + import_descriptors: *mut *mut wasmer_import_descriptors_t, +) { + if module.is_null() { + return; + } + let module = &*(module as *const Module); + let total_imports = module.info().imported_functions.len() + + module.info().imported_tables.len() + + module.info().imported_globals.len() + + module.info().imported_memories.len(); + let mut descriptors: Vec = Vec::with_capacity(total_imports); + + for ( + _index, + ImportName { + namespace_index, + name_index, + }, + ) in &module.info().imported_functions + { + let namespace = module.info().namespace_table.get(*namespace_index); + let name = module.info().name_table.get(*name_index); + descriptors.push(NamedImportDescriptor { + module: namespace.to_string(), + name: name.to_string(), + kind: wasmer_import_export_kind::WASM_FUNCTION, + }); + } + + for ( + _index, + ( + ImportName { + namespace_index, + name_index, + }, + _, + ), + ) in &module.info().imported_tables + { + let namespace = module.info().namespace_table.get(*namespace_index); + let name = module.info().name_table.get(*name_index); + descriptors.push(NamedImportDescriptor { + module: namespace.to_string(), + name: name.to_string(), + kind: wasmer_import_export_kind::WASM_TABLE, + }); + } + + for ( + _index, + ( + ImportName { + namespace_index, + name_index, + }, + _, + ), + ) in &module.info().imported_globals + { + let namespace = module.info().namespace_table.get(*namespace_index); + let name = module.info().name_table.get(*name_index); + descriptors.push(NamedImportDescriptor { + module: namespace.to_string(), + name: name.to_string(), + kind: wasmer_import_export_kind::WASM_GLOBAL, + }); + } + + for ( + _index, + ( + ImportName { + namespace_index, + name_index, + }, + _, + ), + ) in &module.info().imported_memories + { + let namespace = module.info().namespace_table.get(*namespace_index); + let name = module.info().name_table.get(*name_index); + descriptors.push(NamedImportDescriptor { + module: namespace.to_string(), + name: name.to_string(), + kind: wasmer_import_export_kind::WASM_MEMORY, + }); + } + + let named_import_descriptors: Box = + Box::new(NamedImportDescriptors(descriptors)); + *import_descriptors = + Box::into_raw(named_import_descriptors) as *mut wasmer_import_descriptors_t; +} + +pub struct NamedImportDescriptors(Vec); + +/// Frees the memory for the given import descriptors +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub extern "C" fn wasmer_import_descriptors_destroy( + import_descriptors: *mut wasmer_import_descriptors_t, +) { + if !import_descriptors.is_null() { + unsafe { Box::from_raw(import_descriptors as *mut NamedImportDescriptors) }; + } +} + +/// Gets the length of the import descriptors +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub unsafe extern "C" fn wasmer_import_descriptors_len( + exports: *mut wasmer_import_descriptors_t, +) -> c_uint { + if exports.is_null() { + return 0; + } + (*(exports as *mut NamedImportDescriptors)).0.len() as c_uint +} + +/// Gets import descriptor by index +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub unsafe extern "C" fn wasmer_import_descriptors_get( + import_descriptors: *mut wasmer_import_descriptors_t, + idx: c_uint, +) -> *mut wasmer_import_descriptor_t { + if import_descriptors.is_null() { + return ptr::null_mut(); + } + let named_import_descriptors = &mut *(import_descriptors as *mut NamedImportDescriptors); + &mut (*named_import_descriptors).0[idx as usize] as *mut NamedImportDescriptor + as *mut wasmer_import_descriptor_t +} + +/// Gets name for the import descriptor +#[no_mangle] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe extern "C" fn wasmer_import_descriptor_name( + import_descriptor: *mut wasmer_import_descriptor_t, +) -> wasmer_byte_array { + let named_import_descriptor = &*(import_descriptor as *mut NamedImportDescriptor); + wasmer_byte_array { + bytes: named_import_descriptor.name.as_ptr(), + bytes_len: named_import_descriptor.name.len() as u32, + } +} + +/// Gets module name for the import descriptor +#[no_mangle] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe extern "C" fn wasmer_import_descriptor_module_name( + import_descriptor: *mut wasmer_import_descriptor_t, +) -> wasmer_byte_array { + let named_import_descriptor = &*(import_descriptor as *mut NamedImportDescriptor); + wasmer_byte_array { + bytes: named_import_descriptor.module.as_ptr(), + bytes_len: named_import_descriptor.module.len() as u32, + } +} + +/// Gets export descriptor kind +#[no_mangle] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe extern "C" fn wasmer_import_descriptor_kind( + export: *mut wasmer_import_descriptor_t, +) -> wasmer_import_export_kind { + let named_import_descriptor = &*(export as *mut NamedImportDescriptor); + named_import_descriptor.kind.clone() +} + +/// Sets the result parameter to the arity of the params of the wasmer_import_func_t +/// +/// Returns `wasmer_result_t::WASMER_OK` upon success. +/// +/// Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` +/// and `wasmer_last_error_message` to get an error message. +#[no_mangle] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe extern "C" fn wasmer_import_func_params_arity( + func: *const wasmer_import_func_t, + result: *mut u32, +) -> wasmer_result_t { + let export = &*(func as *const Export); + if let Export::Function { ref signature, .. } = *export { + *result = signature.params().len() as u32; + wasmer_result_t::WASMER_OK + } else { + update_last_error(CApiError { + msg: "func ptr error in wasmer_import_func_params_arity".to_string(), + }); + wasmer_result_t::WASMER_ERROR + } +} + +/// Creates new host function, aka imported function. `func` is a +/// function pointer, where the first argument is the famous `vm::Ctx` +/// (in Rust), or `wasmer_instance_context_t` (in C). All arguments +/// must be typed with compatible WebAssembly native types: +/// +/// | WebAssembly type | C/C++ type | +/// | ---------------- | ---------- | +/// | `i32` | `int32_t` | +/// | `i64` | `int64_t` | +/// | `f32` | `float` | +/// | `f64` | `double` | +/// +/// The function pointer must have a lifetime greater than the +/// WebAssembly instance lifetime. +/// +/// The caller owns the object and should call +/// `wasmer_import_func_destroy` to free it. +#[no_mangle] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe extern "C" fn wasmer_import_func_new( + func: extern "C" fn(data: *mut c_void), + params: *const wasmer_value_tag, + params_len: c_uint, + returns: *const wasmer_value_tag, + returns_len: c_uint, +) -> *mut wasmer_import_func_t { + let params: &[wasmer_value_tag] = slice::from_raw_parts(params, params_len as usize); + let params: Vec = params.iter().cloned().map(|x| x.into()).collect(); + let returns: &[wasmer_value_tag] = slice::from_raw_parts(returns, returns_len as usize); + let returns: Vec = returns.iter().cloned().map(|x| x.into()).collect(); + + let export = Box::new(Export::Function { + func: FuncPointer::new(func as _), + ctx: Context::Internal, + signature: Arc::new(FuncSig::new(params, returns)), + }); + Box::into_raw(export) as *mut wasmer_import_func_t +} + +/// Stop the execution of a host function, aka imported function. The +/// function must be used _only_ inside a host function. +/// +/// The pointer to `wasmer_instance_context_t` is received by the host +/// function as its first argument. Just passing it to `ctx` is fine. +/// +/// The error message must have a greater lifetime than the host +/// function itself since the error is read outside the host function +/// with `wasmer_last_error_message`. +/// +/// This function returns `wasmer_result_t::WASMER_ERROR` if `ctx` or +/// `error_message` are null. +/// +/// This function never returns otherwise. +#[no_mangle] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe extern "C" fn wasmer_trap( + ctx: *const wasmer_instance_context_t, + error_message: *const c_char, +) -> wasmer_result_t { + if ctx.is_null() { + update_last_error(CApiError { + msg: "ctx ptr is null in wasmer_trap".to_string(), + }); + + return wasmer_result_t::WASMER_ERROR; + } + + if error_message.is_null() { + update_last_error(CApiError { + msg: "error_message is null in wasmer_trap".to_string(), + }); + + return wasmer_result_t::WASMER_ERROR; + } + + let ctx = &*(ctx as *const Ctx); + let error_message = CStr::from_ptr(error_message).to_str().unwrap(); + + (&*ctx.module) + .runnable_module + .do_early_trap(Box::new(error_message)); // never returns + + // cbindgen does not generate a binding for a function that + // returns `!`. Since we also need to error in some cases, the + // output type of `wasmer_trap` is `wasmer_result_t`. But the OK + // case is not reachable because `do_early_trap` never + // returns. That's a compromise to satisfy the Rust type system, + // cbindgen, and get an acceptable clean code. + #[allow(unreachable_code)] + wasmer_result_t::WASMER_OK +} + +/// Sets the params buffer to the parameter types of the given wasmer_import_func_t +/// +/// Returns `wasmer_result_t::WASMER_OK` upon success. +/// +/// Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` +/// and `wasmer_last_error_message` to get an error message. +#[no_mangle] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe extern "C" fn wasmer_import_func_params( + func: *const wasmer_import_func_t, + params: *mut wasmer_value_tag, + params_len: c_uint, +) -> wasmer_result_t { + let export = &*(func as *const Export); + if let Export::Function { ref signature, .. } = *export { + let params: &mut [wasmer_value_tag] = + slice::from_raw_parts_mut(params, params_len as usize); + for (i, item) in signature.params().iter().enumerate() { + params[i] = item.into(); + } + wasmer_result_t::WASMER_OK + } else { + update_last_error(CApiError { + msg: "func ptr error in wasmer_import_func_params".to_string(), + }); + wasmer_result_t::WASMER_ERROR + } +} + +/// Sets the returns buffer to the parameter types of the given wasmer_import_func_t +/// +/// Returns `wasmer_result_t::WASMER_OK` upon success. +/// +/// Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` +/// and `wasmer_last_error_message` to get an error message. +#[no_mangle] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe extern "C" fn wasmer_import_func_returns( + func: *const wasmer_import_func_t, + returns: *mut wasmer_value_tag, + returns_len: c_uint, +) -> wasmer_result_t { + let export = &*(func as *const Export); + if let Export::Function { ref signature, .. } = *export { + let returns: &mut [wasmer_value_tag] = + slice::from_raw_parts_mut(returns, returns_len as usize); + for (i, item) in signature.returns().iter().enumerate() { + returns[i] = item.into(); + } + wasmer_result_t::WASMER_OK + } else { + update_last_error(CApiError { + msg: "func ptr error in wasmer_import_func_returns".to_string(), + }); + wasmer_result_t::WASMER_ERROR + } +} + +/// Sets the result parameter to the arity of the returns of the wasmer_import_func_t +/// +/// Returns `wasmer_result_t::WASMER_OK` upon success. +/// +/// Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` +/// and `wasmer_last_error_message` to get an error message. +#[no_mangle] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe extern "C" fn wasmer_import_func_returns_arity( + func: *const wasmer_import_func_t, + result: *mut u32, +) -> wasmer_result_t { + let export = &*(func as *const Export); + if let Export::Function { ref signature, .. } = *export { + *result = signature.returns().len() as u32; + wasmer_result_t::WASMER_OK + } else { + update_last_error(CApiError { + msg: "func ptr error in wasmer_import_func_results_arity".to_string(), + }); + wasmer_result_t::WASMER_ERROR + } +} + +/// Frees memory for the given Func +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub extern "C" fn wasmer_import_func_destroy(func: *mut wasmer_import_func_t) { + if !func.is_null() { + unsafe { Box::from_raw(func as *mut Export) }; + } +} + +/// Frees memory of the given ImportObject +#[no_mangle] +pub extern "C" fn wasmer_import_object_destroy(import_object: *mut wasmer_import_object_t) { + if !import_object.is_null() { + unsafe { Box::from_raw(import_object as *mut ImportObject) }; + } +} + +struct NamedImportDescriptor { + module: String, + name: String, + kind: wasmer_import_export_kind, +} diff --git a/lib/runtime-c-api/src/instance.rs b/lib/runtime-c-api/src/instance.rs new file mode 100644 index 000000000000..cf7587198f26 --- /dev/null +++ b/lib/runtime-c-api/src/instance.rs @@ -0,0 +1,694 @@ +//! Instantiate a module, call functions, and read exports. + +use crate::{ + error::{update_last_error, CApiError}, + export::{wasmer_exports_t, wasmer_import_export_kind, NamedExport, NamedExports}, + import::{wasmer_import_t, GLOBAL_IMPORT_OBJECT}, + memory::wasmer_memory_t, + value::{wasmer_value, wasmer_value_t, wasmer_value_tag}, + wasmer_result_t, +}; +use libc::{c_char, c_int, c_void}; +use std::{collections::HashMap, ffi::CStr, ptr, slice}; +use wasmer_runtime::{Ctx, Global, Instance, Memory, Table, Value}; +use wasmer_runtime_core::{ + export::Export, + import::{ImportObject, Namespace}, +}; + +use wasmer_runtime_core::backend::Compiler; +use wasmer_runtime_core::codegen::{MiddlewareChain, StreamingCompiler}; +use crate::metering::OPCODE_COSTS; + +#[cfg(not(feature = "cranelift-backend"))] +use wasmer_middleware_common::metering; + +use wasmer_middleware_common::runtime_breakpoints; +use wasmer_middleware_common::opcode_trace; +use wasmer_middleware_common::opcode_control; + + + +/// Opaque pointer to a `wasmer_runtime::Instance` value in Rust. +/// +/// A `wasmer_runtime::Instance` represents a WebAssembly instance. It +/// is generally generated by the `wasmer_instantiate()` function, or by +/// the `wasmer_module_instantiate()` function for the most common paths. +#[repr(C)] +pub struct wasmer_instance_t; + +/// Opaque pointer to a `wasmer_runtime::Ctx` value in Rust. +/// +/// An instance context is passed to any host function (aka imported +/// function) as the first argument. It is necessary to read the +/// instance data or the memory, respectively with the +/// `wasmer_instance_context_data_get()` function, and the +/// `wasmer_instance_context_memory()` function. +/// +/// It is also possible to get the instance context outside a host +/// function by using the `wasmer_instance_context_get()` +/// function. See also `wasmer_instance_context_data_set()` to set the +/// instance context data. +/// +/// Example: +/// +/// ```c +/// // A host function that prints data from the WebAssembly memory to +/// // the standard output. +/// void print(wasmer_instance_context_t *context, int32_t pointer, int32_t length) { +/// // Use `wasmer_instance_context` to get back the first instance memory. +/// const wasmer_memory_t *memory = wasmer_instance_context_memory(context, 0); +/// +/// // Continue… +/// } +/// ``` +#[repr(C)] +pub struct wasmer_instance_context_t; + +/// Creates a new WebAssembly instance from the given bytes and imports. +/// +/// The result is stored in the first argument `instance` if +/// successful, i.e. when the function returns +/// `wasmer_result_t::WASMER_OK`. Otherwise +/// `wasmer_result_t::WASMER_ERROR` is returned, and +/// `wasmer_last_error_length()` with `wasmer_last_error_message()` must +/// be used to read the error message. +/// +/// The caller is responsible to free the instance with +/// `wasmer_instance_destroy()`. +/// +/// Example: +/// +/// ```c +/// // 1. Read a WebAssembly module from a file. +/// FILE *file = fopen("sum.wasm", "r"); +/// fseek(file, 0, SEEK_END); +/// long bytes_length = ftell(file); +/// uint8_t *bytes = malloc(bytes_length); +/// fseek(file, 0, SEEK_SET); +/// fread(bytes, 1, bytes_length, file); +/// fclose(file); +/// +/// // 2. Declare the imports (here, none). +/// wasmer_import_t imports[] = {}; +/// +/// // 3. Instantiate the WebAssembly module. +/// wasmer_instance_t *instance = NULL; +/// wasmer_result_t result = wasmer_instantiate(&instance, bytes, bytes_length, imports, 0); +/// +/// // 4. Check for errors. +/// if (result != WASMER_OK) { +/// int error_length = wasmer_last_error_length(); +/// char *error = malloc(error_length); +/// wasmer_last_error_message(error, error_length); +/// // Do something with `error`… +/// } +/// +/// // 5. Free the memory! +/// wasmer_instance_destroy(instance); +/// ``` +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub unsafe extern "C" fn wasmer_instantiate( + instance: *mut *mut wasmer_instance_t, + wasm_bytes: *mut u8, + wasm_bytes_len: u32, + imports: *mut wasmer_import_t, + imports_len: c_int, +) -> wasmer_result_t { + if wasm_bytes.is_null() { + update_last_error(CApiError { + msg: "wasm bytes ptr is null".to_string(), + }); + return wasmer_result_t::WASMER_ERROR; + } + let imports: &[wasmer_import_t] = slice::from_raw_parts(imports, imports_len as usize); + let mut import_object = ImportObject::new(); + let mut namespaces = HashMap::new(); + for import in imports { + let module_name = slice::from_raw_parts( + import.module_name.bytes, + import.module_name.bytes_len as usize, + ); + let module_name = if let Ok(s) = std::str::from_utf8(module_name) { + s + } else { + update_last_error(CApiError { + msg: "error converting module name to string".to_string(), + }); + return wasmer_result_t::WASMER_ERROR; + }; + let import_name = slice::from_raw_parts( + import.import_name.bytes, + import.import_name.bytes_len as usize, + ); + let import_name = if let Ok(s) = std::str::from_utf8(import_name) { + s + } else { + update_last_error(CApiError { + msg: "error converting import_name to string".to_string(), + }); + return wasmer_result_t::WASMER_ERROR; + }; + + let namespace = namespaces.entry(module_name).or_insert_with(Namespace::new); + + // TODO check that tag is actually in bounds here + let export = match import.tag { + wasmer_import_export_kind::WASM_MEMORY => { + let mem = import.value.memory as *mut Memory; + Export::Memory((&*mem).clone()) + } + wasmer_import_export_kind::WASM_FUNCTION => { + let func_export = import.value.func as *mut Export; + (&*func_export).clone() + } + wasmer_import_export_kind::WASM_GLOBAL => { + let global = import.value.global as *mut Global; + Export::Global((&*global).clone()) + } + wasmer_import_export_kind::WASM_TABLE => { + let table = import.value.table as *mut Table; + Export::Table((&*table).clone()) + } + }; + namespace.insert(import_name, export); + } + for (module_name, namespace) in namespaces.into_iter() { + import_object.register(module_name, namespace); + } + + let bytes: &[u8] = slice::from_raw_parts_mut(wasm_bytes, wasm_bytes_len as usize); + let result = wasmer_runtime::instantiate(bytes, &import_object); + let new_instance = match result { + Ok(instance) => instance, + Err(error) => { + update_last_error(error); + return wasmer_result_t::WASMER_ERROR; + } + }; + *instance = Box::into_raw(Box::new(new_instance)) as *mut wasmer_instance_t; + wasmer_result_t::WASMER_OK +} + + +#[repr(C)] +pub struct wasmer_import_object_t; + +#[repr(C)] +pub struct wasmer_compilation_options_t; + +pub struct CompilationOptions { + pub gas_limit: u64, + pub unmetered_locals: usize, + pub max_memory_grow: usize, + pub max_memory_grow_delta: usize, + pub opcode_trace: bool, + pub metering: bool, + pub runtime_breakpoints: bool, +} + +#[allow(clippy::cast_ptr_alignment)] +#[cfg(feature = "metering")] +#[no_mangle] +pub unsafe extern "C" fn wasmer_instantiate_with_options( + instance: *mut *mut wasmer_instance_t, + wasm_bytes: *mut u8, + wasm_bytes_len: u32, + options: *const wasmer_compilation_options_t, +) -> wasmer_result_t { + if wasm_bytes.is_null() { + update_last_error(CApiError { + msg: "wasm bytes ptr is null".to_string(), + }); + return wasmer_result_t::WASMER_ERROR; + } + + let bytes: &[u8] = slice::from_raw_parts_mut(wasm_bytes, wasm_bytes_len as usize); + let options: &CompilationOptions = &*(options as *const CompilationOptions); + let compiler_chain_generator = prepare_middleware_chain_generator(&options); + let compiler = get_compiler(compiler_chain_generator); + let result_compilation = wasmer_runtime_core::compile_with(bytes, &compiler); + let new_module = match result_compilation { + Ok(module) => module, + Err(_) => { + update_last_error(CApiError { msg: "compile error".to_string() }); + return wasmer_result_t::WASMER_ERROR; + } + }; + + let import_object: &mut ImportObject = &mut *(GLOBAL_IMPORT_OBJECT as *mut ImportObject); + let result_instantiation = new_module.instantiate(&import_object); + let mut new_instance = match result_instantiation { + Ok(instance) => instance, + Err(error) => { + update_last_error(error); + return wasmer_result_t::WASMER_ERROR; + } + }; + metering::set_points_limit(&mut new_instance, options.gas_limit); + *instance = Box::into_raw(Box::new(new_instance)) as *mut wasmer_instance_t; + wasmer_result_t::WASMER_OK +} + +pub unsafe fn prepare_middleware_chain_generator( + options: &CompilationOptions + ) -> impl Fn() -> MiddlewareChain + '_ { + let options = options.clone(); + + let chain_generator = move || { + let mut chain = MiddlewareChain::new(); + + if options.metering { + #[cfg(feature = "metering")] + chain.push(metering::Metering::new( + &OPCODE_COSTS, + options.unmetered_locals + )); + } + + chain.push(opcode_control::OpcodeControl::new( + options.max_memory_grow, + options.max_memory_grow_delta + )); + + // The RuntimeBreakpointHandler must be the last middleware in the chain (OpcodeTracer is + // an exception since it does not alter the opcodes meaningfully. + if options.runtime_breakpoints { + chain.push(runtime_breakpoints::RuntimeBreakpointHandler::new()); + } + + if options.opcode_trace { + chain.push(opcode_trace::OpcodeTracer::new()); + }; + + chain + }; + + chain_generator +} + +pub unsafe fn get_compiler(chain_generator: impl Fn() -> MiddlewareChain) -> impl Compiler { + #[cfg(feature = "llvm-backend")] + use wasmer_llvm_backend::ModuleCodeGenerator as MeteredMCG; + + #[cfg(feature = "singlepass-backend")] + use wasmer_singlepass_backend::ModuleCodeGenerator as MeteredMCG; + + #[cfg(feature = "cranelift-backend")] + use wasmer_clif_backend::CraneliftModuleCodeGenerator as MeteredMCG; + + let compiler: StreamingCompiler = StreamingCompiler::new(chain_generator); + compiler +} + + +/// Returns the instance context. Learn more by looking at the +/// `wasmer_instance_context_t` struct. +/// +/// This function returns `null` if `instance` is a null pointer. +/// +/// Example: +/// +/// ```c +/// const wasmer_instance_context_get *context = wasmer_instance_context_get(instance); +/// my_data *data = (my_data *) wasmer_instance_context_data_get(context); +/// // Do something with `my_data`. +/// ``` +/// +/// It is often useful with `wasmer_instance_context_data_set()`. +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub extern "C" fn wasmer_instance_context_get( + instance: *mut wasmer_instance_t, +) -> *const wasmer_instance_context_t { + if instance.is_null() { + return ptr::null() as _; + } + + let instance = unsafe { &*(instance as *const Instance) }; + let context: *const Ctx = instance.context() as *const _; + + context as *const wasmer_instance_context_t +} + +/// Verifies whether the specified function name is imported by the given instance. +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub unsafe extern "C" fn wasmer_instance_is_function_imported( + instance: *mut wasmer_instance_t, + name: *const c_char, +) -> bool { + if instance.is_null() { + return false; + } + + if name.is_null() { + return false; + } + + let instance = &*(instance as *const Instance); + + let func_name_c = CStr::from_ptr(name); + let func_name_r = func_name_c.to_str().unwrap(); + + let module = instance.module(); + + let functions = module.info().name_table.to_vec(); + + functions.contains(&func_name_r) +} + +/// Calls an exported function of a WebAssembly instance by `name` +/// with the provided parameters. The exported function results are +/// stored on the provided `results` pointer. +/// +/// This function returns `wasmer_result_t::WASMER_OK` upon success, +/// `wasmer_result_t::WASMER_ERROR` otherwise. You can use +/// `wasmer_last_error_message()` to get the generated error message. +/// +/// Potential errors are the following: +/// +/// * `instance` is a null pointer, +/// * `name` is a null pointer, +/// * `params` is a null pointer. +/// +/// Example of calling an exported function that needs two parameters, and returns one value: +/// +/// ```c +/// // First argument. +/// wasmer_value_t argument_one = { +/// .tag = WASM_I32, +/// .value.I32 = 3, +/// }; +/// +/// // Second argument. +/// wasmer_value_t argument_two = { +/// .tag = WASM_I32, +/// .value.I32 = 4, +/// }; +/// +/// // First result. +/// wasmer_value_t result_one; +/// +/// // All arguments and results. +/// wasmer_value_t arguments[] = {argument_one, argument_two}; +/// wasmer_value_t results[] = {result_one}; +/// +/// wasmer_result_t call_result = wasmer_instance_call( +/// instance, // instance pointer +/// "sum", // the exported function name +/// arguments, // the arguments +/// 2, // the number of arguments +/// results, // the results +/// 1 // the number of results +/// ); +/// +/// if (call_result == WASMER_OK) { +/// printf("Result is: %d\n", results[0].value.I32); +/// } +/// ``` +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub unsafe extern "C" fn wasmer_instance_call( + instance: *mut wasmer_instance_t, + name: *const c_char, + params: *const wasmer_value_t, + params_len: u32, + results: *mut wasmer_value_t, + results_len: u32, +) -> wasmer_result_t { + if instance.is_null() { + update_last_error(CApiError { + msg: "instance ptr is null".to_string(), + }); + + return wasmer_result_t::WASMER_ERROR; + } + + if name.is_null() { + update_last_error(CApiError { + msg: "name ptr is null".to_string(), + }); + + return wasmer_result_t::WASMER_ERROR; + } + + if params.is_null() { + update_last_error(CApiError { + msg: "params ptr is null".to_string(), + }); + + return wasmer_result_t::WASMER_ERROR; + } + + let params: &[wasmer_value_t] = slice::from_raw_parts(params, params_len as usize); + let params: Vec = params.iter().cloned().map(|x| x.into()).collect(); + + let func_name_c = CStr::from_ptr(name); + let func_name_r = func_name_c.to_str().unwrap(); + + let results: &mut [wasmer_value_t] = slice::from_raw_parts_mut(results, results_len as usize); + let instance = &mut *(instance as *mut Instance); + + wasmer_middleware_common::opcode_trace::reset_opcodetracer_last_location(instance); + let result = instance.call(func_name_r, ¶ms[..]); + + let result = match result { + Ok(results_vec) => { + if !results_vec.is_empty() { + let ret = match results_vec[0] { + Value::I32(x) => wasmer_value_t { + tag: wasmer_value_tag::WASM_I32, + value: wasmer_value { I32: x }, + }, + Value::I64(x) => wasmer_value_t { + tag: wasmer_value_tag::WASM_I64, + value: wasmer_value { I64: x }, + }, + Value::F32(x) => wasmer_value_t { + tag: wasmer_value_tag::WASM_F32, + value: wasmer_value { F32: x }, + }, + Value::F64(x) => wasmer_value_t { + tag: wasmer_value_tag::WASM_F64, + value: wasmer_value { F64: x }, + }, + Value::V128(_) => unimplemented!("calling function with V128 parameter"), + }; + results[0] = ret; + } + wasmer_result_t::WASMER_OK + } + Err(err) => { + update_last_error(err); + wasmer_result_t::WASMER_ERROR + } + }; + + let last_opcode_location = wasmer_middleware_common::opcode_trace::get_opcodetracer_last_location(instance); + if last_opcode_location > 0 { + let imported_functions = instance.module.info.name_table.to_vec(); + for i in 0..imported_functions.len() { + println!("Import {}\t{}", i, imported_functions[i]); + } + + for (k, v) in instance.module.info.exports.iter() { + println!("Export {:?}\t{}", v, k); + } + + println!("wasmer_instance_call OPCODE_LAST_LOCATION = {}", last_opcode_location); + } + + result +} + +/// Gets all the exports of the given WebAssembly instance. +/// + +/// This function stores a Rust vector of exports into `exports` as an +/// opaque pointer of kind `wasmer_exports_t`. +/// +/// As is, you can do anything with `exports` except using the +/// companion functions, like `wasmer_exports_len()`, +/// `wasmer_exports_get()` or `wasmer_export_kind()`. See the example below. +/// +/// **Warning**: The caller owns the object and should call +/// `wasmer_exports_destroy()` to free it. +/// +/// Example: +/// +/// ```c +/// // Get the exports. +/// wasmer_exports_t *exports = NULL; +/// wasmer_instance_exports(instance, &exports); +/// +/// // Get the number of exports. +/// int exports_length = wasmer_exports_len(exports); +/// printf("Number of exports: %d\n", exports_length); +/// +/// // Read the first export. +/// wasmer_export_t *export = wasmer_exports_get(exports, 0); +/// +/// // Get the kind of the export. +/// wasmer_import_export_kind export_kind = wasmer_export_kind(export); +/// +/// // Assert it is a function (why not). +/// assert(export_kind == WASM_FUNCTION); +/// +/// // Read the export name. +/// wasmer_byte_array name_bytes = wasmer_export_name(export); +/// +/// assert(name_bytes.bytes_len == sizeof("sum") - 1); +/// assert(memcmp(name_bytes.bytes, "sum", sizeof("sum") - 1) == 0); +/// +/// // Destroy the exports. +/// wasmer_exports_destroy(exports); +/// ``` +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub unsafe extern "C" fn wasmer_instance_exports( + instance: *mut wasmer_instance_t, + exports: *mut *mut wasmer_exports_t, +) { + if instance.is_null() { + return; + } + + let instance_ref = &mut *(instance as *mut Instance); + let mut exports_vec: Vec = Vec::with_capacity(instance_ref.exports().count()); + + for (name, export) in instance_ref.exports() { + exports_vec.push(NamedExport { + name: name.clone(), + export: export.clone(), + instance: instance as *mut Instance, + }); + } + + let named_exports: Box = Box::new(NamedExports(exports_vec)); + + *exports = Box::into_raw(named_exports) as *mut wasmer_exports_t; +} + +/// Sets the data that can be hold by an instance context. +/// +/// An instance context (represented by the opaque +/// `wasmer_instance_context_t` structure) can hold user-defined +/// data. This function sets the data. This function is complementary +/// of `wasmer_instance_context_data_get()`. +/// +/// This function does nothing if `instance` is a null pointer. +/// +/// Example: +/// +/// ```c +/// // Define your own data. +/// typedef struct { +/// // … +/// } my_data; +/// +/// // Allocate them and set them on the given instance. +/// my_data *data = malloc(sizeof(my_data)); +/// data->… = …; +/// wasmer_instance_context_data_set(instance, (void*) my_data); +/// +/// // You can read your data. +/// { +/// my_data *data = (my_data*) wasmer_instance_context_data_get(wasmer_instance_context_get(instance)); +/// // … +/// } +/// ``` +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub extern "C" fn wasmer_instance_context_data_set( + instance: *mut wasmer_instance_t, + data_ptr: *mut c_void, +) { + if instance.is_null() { + return; + } + + let instance = unsafe { &mut *(instance as *mut Instance) }; + + instance.context_mut().data = data_ptr; +} + +/// Gets the `memory_idx`th memory of the instance. +/// +/// Note that the index is always `0` until multiple memories are supported. +/// +/// This function is mostly used inside host functions (aka imported +/// functions) to read the instance memory. +/// +/// Example of a _host function_ that reads and prints a string based on a pointer and a length: +/// +/// ```c +/// void print_string(const wasmer_instance_context_t *context, int32_t pointer, int32_t length) { +/// // Get the 0th memory. +/// const wasmer_memory_t *memory = wasmer_instance_context_memory(context, 0); +/// +/// // Get the memory data as a pointer. +/// uint8_t *memory_bytes = wasmer_memory_data(memory); +/// +/// // Print what we assumed to be a string! +/// printf("%.*s", length, memory_bytes + pointer); +/// } +/// ``` +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub extern "C" fn wasmer_instance_context_memory( + ctx: *const wasmer_instance_context_t, + _memory_idx: u32, +) -> *const wasmer_memory_t { + let ctx = unsafe { &*(ctx as *const Ctx) }; + let memory = ctx.memory(0); + memory as *const Memory as *const wasmer_memory_t +} + +/// Gets the data that can be hold by an instance. +/// +/// This function is complementary of +/// `wasmer_instance_context_data_set()`. Please read its +/// documentation. You can also read the documentation of +/// `wasmer_instance_context_t` to get other examples. +/// +/// This function returns nothing if `ctx` is a null pointer. +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub extern "C" fn wasmer_instance_context_data_get( + ctx: *const wasmer_instance_context_t, +) -> *mut c_void { + if ctx.is_null() { + return ptr::null_mut() as _; + } + + let ctx = unsafe { &*(ctx as *const Ctx) }; + + ctx.data +} + +/// Frees memory for the given `wasmer_instance_t`. +/// +/// Check the `wasmer_instantiate()` function to get a complete +/// example. +/// +/// If `instance` is a null pointer, this function does nothing. +/// +/// Example: +/// +/// ```c +/// // Get an instance. +/// wasmer_instance_t *instance = NULL; +/// wasmer_instantiate(&instance, bytes, bytes_length, imports, 0); +/// +/// // Destroy the instance. +/// wasmer_instance_destroy(instance); +/// ``` +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub extern "C" fn wasmer_instance_destroy(instance: *mut wasmer_instance_t) { + if !instance.is_null() { + unsafe { Box::from_raw(instance as *mut Instance) }; + } +} diff --git a/lib/runtime-c-api/src/instance_cache.rs b/lib/runtime-c-api/src/instance_cache.rs new file mode 100644 index 000000000000..58f1e074dd59 --- /dev/null +++ b/lib/runtime-c-api/src/instance_cache.rs @@ -0,0 +1,206 @@ +use crate::{ + error::{update_last_error, CApiError}, + instance::{wasmer_instance_t, wasmer_compilation_options_t, CompilationOptions, prepare_middleware_chain_generator, get_compiler}, + wasmer_result_t, +}; + +use rkyv::{ + Deserialize as RkyvDeserialize, + ser::Serializer, + ser::serializers::AllocSerializer, +}; + +use wasmer_runtime_core::{ + cache::{Artifact, Error as CacheError}, + import::ImportObject, +}; +use std::slice; +use crate::import::GLOBAL_IMPORT_OBJECT; + +#[cfg(not(feature = "cranelift-backend"))] +use wasmer_middleware_common::metering; + +#[cfg(feature = "singlepass-backend")] +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub unsafe extern "C" fn wasmer_instance_enable_rkyv() { + wasmer_singlepass_backend::USE_RKYV_SERIALIZATION = true; +} + +#[cfg(feature = "singlepass-backend")] +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub unsafe extern "C" fn wasmer_instance_disable_rkyv() { + wasmer_singlepass_backend::USE_RKYV_SERIALIZATION = false; +} + +#[cfg(feature = "singlepass-backend")] +pub unsafe fn is_rkyv_enabled() -> bool { + wasmer_singlepass_backend::USE_RKYV_SERIALIZATION +} + +#[cfg(not(feature = "singlepass-backend"))] +pub unsafe fn is_rkyv_enabled() -> bool { + false +} + + +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub unsafe extern "C" fn wasmer_instance_cache( + instance: *mut wasmer_instance_t, + cache_bytes: *mut *const u8, + cache_len: *mut u32, +) -> wasmer_result_t { + if instance.is_null() { + update_last_error(CApiError { + msg: "null instance".to_string(), + }); + return wasmer_result_t::WASMER_ERROR; + } + + let instance = &mut *(instance as *mut wasmer_runtime::Instance); + let module = instance.module(); + + match module.cache() { + Err(error) => { + update_last_error(CApiError { + msg: format!("wasmer_instance_cache: artifact creation failed: {:?}", error), + }); + return wasmer_result_t::WASMER_ERROR; + } + Ok(artifact) => { + match serialize_artifact(artifact) { + Err(error) => { + update_last_error(CApiError { + msg: format!("wasmer_instance_cache: artifact serialization failed: {:?}", error), + }); + return wasmer_result_t::WASMER_ERROR; + } + Ok(bytes) => { + *cache_bytes = bytes.as_ptr(); + *cache_len = bytes.len() as u32; + std::mem::forget(bytes); + } + } + } + }; + + wasmer_result_t::WASMER_OK +} + +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub unsafe extern "C" fn wasmer_instance_from_cache( + instance: *mut *mut wasmer_instance_t, + cache_bytes: *mut u8, + cache_len: u32, + options: *const wasmer_compilation_options_t, +) -> wasmer_result_t { + if cache_bytes.is_null() { + update_last_error(CApiError { + msg: "cache bytes ptr is null".to_string(), + }); + return wasmer_result_t::WASMER_ERROR; + } + + let bytes: &[u8] = slice::from_raw_parts(cache_bytes, cache_len as usize); + let options: &CompilationOptions = &*(options as *const CompilationOptions); + let compiler_chain_generator = prepare_middleware_chain_generator(&options); + let compiler = get_compiler(compiler_chain_generator); + + let artifact = match deserialize_artifact(bytes) { + Ok(deserialized_artifact) => deserialized_artifact, + Err(_) => { + update_last_error(CApiError { + msg: "wasmer_instance_from_cache: artifact deserialization failed".to_string(), + }); + return wasmer_result_t::WASMER_ERROR; + } + }; + + let new_module = match wasmer_runtime_core::load_cache_with(artifact, &compiler) { + Ok(deserialized_module) => { + deserialized_module + } + Err(_) => { + update_last_error(CApiError { + msg: "wasmer_instance_from_cache: artifact instantiation into module failed".to_string(), + }); + return wasmer_result_t::WASMER_ERROR; + } + }; + + let import_object: &mut ImportObject = &mut *(GLOBAL_IMPORT_OBJECT as *mut ImportObject); + let result_instantiation = new_module.instantiate(&import_object); + let mut new_instance = match result_instantiation { + Ok(instance) => instance, + Err(error) => { + update_last_error(error); + return wasmer_result_t::WASMER_ERROR; + } + }; + metering::set_points_limit(&mut new_instance, options.gas_limit); + *instance = Box::into_raw(Box::new(new_instance)) as *mut wasmer_instance_t; + wasmer_result_t::WASMER_OK +} + +#[cfg(feature = "singlepass-backend")] +fn serialize_artifact(artifact: Artifact) -> Result, CacheError> { + let serializer = match unsafe { is_rkyv_enabled() } { + true => serialize_artifact_with_rkyv, + false => serialize_artifact_with_serde, + }; + + serializer(artifact).into() +} + +#[cfg(not(feature = "singlepass-backend"))] +fn serialize_artifact(artifact: Artifact) -> Result, CacheError> { + serialize_artifact_with_serde(artifact).into() +} + +#[cfg(feature = "singlepass-backend")] +fn serialize_artifact_with_rkyv(artifact: Artifact) -> Result, CacheError> { + let mut serializer = AllocSerializer::<4096>::default(); + serializer.serialize_value(&artifact).unwrap(); + let serialized = serializer.into_serializer().into_inner().into_boxed_slice(); + if serialized.is_empty() { + return Err(CacheError::SerializeError("rkyv serialization failed".to_string())); + } + + Ok(serialized) +} + +fn serialize_artifact_with_serde(artifact: Artifact) -> Result, CacheError> { + match artifact.serialize() { + Ok(serialized) => Ok(serialized.into_boxed_slice()), + Err(error) => Err(error), + } +} + +#[cfg(feature = "singlepass-backend")] +fn deserialize_artifact(bytes: &[u8]) -> Result { + let deserializer = match unsafe { is_rkyv_enabled() } { + true => deserialize_artifact_with_rkyv, + false => deserialize_artifact_with_serde, + }; + + deserializer(bytes) +} + +#[cfg(not(feature = "singlepass-backend"))] +fn deserialize_artifact(bytes: &[u8]) -> Result { + deserialize_artifact_with_serde(bytes) +} + +#[cfg(feature = "singlepass-backend")] +fn deserialize_artifact_with_rkyv(bytes: &[u8]) -> Result { + let archived = unsafe { rkyv::archived_root::(&bytes[..]) }; + let artifact: Artifact = RkyvDeserialize::::deserialize(archived, &mut rkyv::Infallible).unwrap(); + Ok(artifact) +} + +fn deserialize_artifact_with_serde(bytes: &[u8]) -> Result { + Artifact::deserialize(bytes) +} diff --git a/lib/runtime-c-api/src/lib.rs b/lib/runtime-c-api/src/lib.rs new file mode 100644 index 000000000000..62690d66cbfd --- /dev/null +++ b/lib/runtime-c-api/src/lib.rs @@ -0,0 +1,188 @@ +#![doc(html_favicon_url = "https://wasmer.io/static/icons/favicon.ico")] +#![doc(html_logo_url = "https://avatars3.githubusercontent.com/u/44205449?s=200&v=4")] + +//! # Wasmer Runtime C API +//! +//! Wasmer is a standalone JIT WebAssembly runtime, aiming to be fully +//! compatible with Emscripten, Rust and Go. [Learn +//! more](https://github.com/wasmerio/wasmer). +//! +//! This crate exposes a C and C++ API for the Wasmer runtime. +//! +//! # Usage +//! +//! The C and C++ header files can be found in the source tree of this +//! crate, respectively [`wasmer.h`][wasmer_h] and +//! [`wasmer.hh`][wasmer_hh]. They are automatically generated, and always +//! up-to-date in this repository. +//! +//! Here is a simple example to use the C API: +//! +//! ```c +//! #include +//! #include "wasmer.h" +//! #include +//! #include +//! +//! int main() +//! { +//! // Read the Wasm file bytes. +//! FILE *file = fopen("sum.wasm", "r"); +//! fseek(file, 0, SEEK_END); +//! long len = ftell(file); +//! uint8_t *bytes = malloc(len); +//! fseek(file, 0, SEEK_SET); +//! fread(bytes, 1, len, file); +//! fclose(file); +//! +//! // Prepare the imports. +//! wasmer_import_t imports[] = {}; +//! +//! // Instantiate! +//! wasmer_instance_t *instance = NULL; +//! wasmer_result_t instantiation_result = wasmer_instantiate(&instance, bytes, len, imports, 0); +//! +//! assert(instantiation_result == WASMER_OK); +//! +//! // Let's call a function. +//! // Start by preparing the arguments. +//! +//! // Value of argument #1 is `7i32`. +//! wasmer_value_t argument_one; +//! argument_one.tag = WASM_I32; +//! argument_one.value.I32 = 7; +//! +//! // Value of argument #2 is `8i32`. +//! wasmer_value_t argument_two; +//! argument_two.tag = WASM_I32; +//! argument_two.value.I32 = 8; +//! +//! // Prepare the arguments. +//! wasmer_value_t arguments[] = {argument_one, argument_two}; +//! +//! // Prepare the return value. +//! wasmer_value_t result_one; +//! wasmer_value_t results[] = {result_one}; +//! +//! // Call the `sum` function with the prepared arguments and the return value. +//! wasmer_result_t call_result = wasmer_instance_call(instance, "sum", arguments, 2, results, 1); +//! +//! // Let's display the result. +//! printf("Call result: %d\n", call_result); +//! printf("Result: %d\n", results[0].value.I32); +//! +//! // `sum(7, 8) == 15`. +//! assert(results[0].value.I32 == 15); +//! assert(call_result == WASMER_OK); +//! +//! wasmer_instance_destroy(instance); +//! +//! return 0; +//! } +//! ``` +//! +//! [wasmer_h]: ./wasmer.h +//! [wasmer_hh]: ./wasmer.hh +#![deny( + dead_code, + unused_imports, + unused_variables, + unused_unsafe, + unreachable_patterns +)] +extern crate wasmer_runtime; +extern crate wasmer_runtime_core; + +pub mod error; +pub mod export; +pub mod global; +pub mod import; +pub mod instance; +pub mod memory; + +#[cfg(feature = "metering")] +pub mod metering; + +pub mod instance_cache; +pub mod runtime_breakpoints; +pub mod signals; + +pub mod module; +pub mod table; +// `not(target_family = "windows")` is simpler than `unix`. See build.rs +// if you want to change the meaning of these `cfg`s in the header file. +#[cfg(all(not(target_family = "windows"), target_arch = "x86_64"))] +pub mod trampoline; +pub mod value; + +/// The `wasmer_result_t` enum is a type that represents either a +/// success, or a failure. +#[allow(non_camel_case_types)] +#[repr(C)] +pub enum wasmer_result_t { + /// Represents a success. + WASMER_OK = 1, + + /// Represents a failure. + WASMER_ERROR = 2, +} + +/// The `wasmer_limits_t` struct is a type that describes a memory +/// options. See the `wasmer_memory_t` struct or the +/// `wasmer_memory_new()` function to get more information. +#[repr(C)] +pub struct wasmer_limits_t { + /// The minimum number of allowed pages. + pub min: u32, + + /// The maximum number of allowed pages. + pub max: wasmer_limit_option_t, +} + +/// The `wasmer_limit_option_t` struct represents an optional limit +/// for `wasmer_limits_t`. +#[repr(C)] +pub struct wasmer_limit_option_t { + /// Whether the limit is set. + pub has_some: bool, + + /// The limit value. + pub some: u32, +} + +#[repr(C)] +pub struct wasmer_byte_array { + pub bytes: *const u8, + pub bytes_len: u32, +} + +impl wasmer_byte_array { + /// Get the data as a slice + pub unsafe fn as_slice<'a>(&self) -> &'a [u8] { + get_slice_checked(self.bytes, self.bytes_len as usize) + } + + /// Copy the data into an owned Vec + pub unsafe fn as_vec(&self) -> Vec { + let mut out = Vec::with_capacity(self.bytes_len as usize); + out.extend_from_slice(self.as_slice()); + + out + } + + /// Read the data as a &str, returns an error if the string is not valid UTF8 + pub unsafe fn as_str<'a>(&self) -> Result<&'a str, std::str::Utf8Error> { + std::str::from_utf8(self.as_slice()) + } +} + +/// Gets a slice from a pointer and a length, returning an empty slice if the +/// pointer is null +#[inline] +pub(crate) unsafe fn get_slice_checked<'a, T>(ptr: *const T, len: usize) -> &'a [T] { + if ptr.is_null() { + &[] + } else { + std::slice::from_raw_parts(ptr, len) + } +} diff --git a/lib/runtime-c-api/src/metering.rs b/lib/runtime-c-api/src/metering.rs new file mode 100644 index 000000000000..8feb1eb05a2e --- /dev/null +++ b/lib/runtime-c-api/src/metering.rs @@ -0,0 +1,192 @@ +use crate::{ + error::{update_last_error, CApiError}, + instance::wasmer_instance_t, + module::wasmer_module_t, + wasmer_result_t, +}; +use std::slice; + +#[cfg(feature = "metering")] +use wasmer_runtime_core::backend::Compiler; + +#[cfg(not(feature = "cranelift-backend"))] +use wasmer_middleware_common::metering; + +pub const OPCODE_COUNT: usize = 448; +pub static mut OPCODE_COSTS: [u32; OPCODE_COUNT] = [0; OPCODE_COUNT]; + +#[allow(clippy::cast_ptr_alignment)] +#[cfg(feature = "metering")] +#[no_mangle] +pub unsafe extern "C" fn wasmer_set_opcode_costs( + opcode_costs_pointer: *const u32, +) { + OPCODE_COSTS.copy_from_slice(slice::from_raw_parts(opcode_costs_pointer, OPCODE_COUNT)); +} + + +// returns gas used +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +#[cfg(feature = "metering")] +pub unsafe extern "C" fn wasmer_instance_get_points_used(instance: *mut wasmer_instance_t) -> u64 { + if instance.is_null() { + return 0; + } + let instance = &*(instance as *const wasmer_runtime::Instance); + let points = metering::get_points_used(instance); + points +} + +// sets gas used +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +#[cfg(feature = "metering")] +pub unsafe extern "C" fn wasmer_instance_set_points_used( + instance: *mut wasmer_instance_t, + new_gas: u64, +) { + if instance.is_null() { + return; + } + let instance = &mut *(instance as *mut wasmer_runtime::Instance); + metering::set_points_used(instance, new_gas) +} + +// sets gas limit +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +#[cfg(feature = "metering")] +pub unsafe extern "C" fn wasmer_instance_set_points_limit( + instance: *mut wasmer_instance_t, + limit: u64, +) { + if instance.is_null() { + return; + } + let instance = &mut *(instance as *mut wasmer_runtime::Instance); + metering::set_points_limit(instance, limit) +} + + +/// Creates a new Module with gas limit from the given wasm bytes. +/// +/// Returns `wasmer_result_t::WASMER_OK` upon success. +/// +/// Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` +/// and `wasmer_last_error_message` to get an error message. +#[allow(clippy::cast_ptr_alignment)] +#[cfg(feature = "metering")] +#[no_mangle] +pub unsafe extern "C" fn wasmer_compile_with_gas_metering( + module: *mut *mut wasmer_module_t, + wasm_bytes: *mut u8, + wasm_bytes_len: u32, +) -> wasmer_result_t { + if module.is_null() { + update_last_error(CApiError { + msg: "module is null".to_string(), + }); + return wasmer_result_t::WASMER_ERROR; + } + if wasm_bytes.is_null() { + update_last_error(CApiError { + msg: "wasm bytes is null".to_string(), + }); + return wasmer_result_t::WASMER_ERROR; + } + + let compiler = get_metered_compiler(); + + let bytes: &[u8] = slice::from_raw_parts_mut(wasm_bytes, wasm_bytes_len as usize); + let result = wasmer_runtime_core::compile_with(bytes, &compiler); + let new_module = match result { + Ok(instance) => instance, + Err(_) => { + update_last_error(CApiError { + msg: "compile error".to_string(), + }); + return wasmer_result_t::WASMER_ERROR; + } + }; + *module = Box::into_raw(Box::new(new_module)) as *mut wasmer_module_t; + wasmer_result_t::WASMER_OK +} + +#[cfg(feature = "metering")] +unsafe fn get_metered_compiler() -> impl Compiler { + use wasmer_runtime_core::codegen::{MiddlewareChain, StreamingCompiler}; + + #[cfg(feature = "llvm-backend")] + use wasmer_llvm_backend::ModuleCodeGenerator as MeteredMCG; + + #[cfg(feature = "singlepass-backend")] + use wasmer_singlepass_backend::ModuleCodeGenerator as MeteredMCG; + + #[cfg(feature = "cranelift-backend")] + use wasmer_clif_backend::CraneliftModuleCodeGenerator as MeteredMCG; + + use wasmer_middleware_common::runtime_breakpoints; + + let c: StreamingCompiler = StreamingCompiler::new(move || { + let mut chain = MiddlewareChain::new(); + + chain.push(metering::Metering::new(&OPCODE_COSTS, 0)); + chain.push(runtime_breakpoints::RuntimeBreakpointHandler::new()); + + chain + }); + c +} + + +/*** placeholder implementation if metering feature off ***/ + +// Without metering, wasmer_compile_with_gas_metering is a copy of wasmer_compile +#[cfg(not(feature = "metering"))] +#[no_mangle] +pub unsafe extern "C" fn wasmer_compile_with_gas_metering( + module: *mut *mut wasmer_module_t, + wasm_bytes: *mut u8, + wasm_bytes_len: u32, +) -> wasmer_result_t { + if module.is_null() { + update_last_error(CApiError { + msg: "module is null".to_string(), + }); + return wasmer_result_t::WASMER_ERROR; + } + if wasm_bytes.is_null() { + update_last_error(CApiError { + msg: "wasm bytes is null".to_string(), + }); + return wasmer_result_t::WASMER_ERROR; + } + + let bytes: &[u8] = slice::from_raw_parts_mut(wasm_bytes, wasm_bytes_len as usize); + // TODO: this implicitly uses default_compiler() is that proper? maybe we override default_compiler + let result = wasmer_runtime::compile(bytes); + let new_module = match result { + Ok(instance) => instance, + Err(error) => { + update_last_error(error); + return wasmer_result_t::WASMER_ERROR; + } + }; + *module = Box::into_raw(Box::new(new_module)) as *mut wasmer_module_t; + wasmer_result_t::WASMER_OK +} + +// returns gas used -- placeholder implementation, when "metering" is disabled +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +#[cfg(not(feature = "metering"))] +pub unsafe extern "C" fn wasmer_instance_get_points_used(_: *mut wasmer_instance_t) -> u64 { + 0 +} + +// sets gas used -- placeholder implementation, when "metering" is disabled +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +#[cfg(not(feature = "metering"))] +pub unsafe extern "C" fn wasmer_instance_set_points_used(_: *mut wasmer_instance_t, _: u64) {} diff --git a/lib/runtime-c-api/src/module.rs b/lib/runtime-c-api/src/module.rs new file mode 100644 index 000000000000..27f82d0b0ac7 --- /dev/null +++ b/lib/runtime-c-api/src/module.rs @@ -0,0 +1,329 @@ +//! Compile, validate, instantiate, serialize, and destroy modules. + +use crate::{ + error::{update_last_error, CApiError}, + export::wasmer_import_export_kind, + import::{wasmer_import_object_t, wasmer_import_t}, + instance::wasmer_instance_t, + wasmer_byte_array, wasmer_result_t, +}; +use libc::c_int; +use std::{collections::HashMap, slice}; +use wasmer_runtime::{ + compile, default_compiler, Global, ImportObject, Instance, Memory, Module, Table, +}; +use wasmer_runtime_core::{cache::Artifact, export::Export, import::Namespace, load_cache_with}; + +#[repr(C)] +pub struct wasmer_module_t; + +#[repr(C)] +pub struct wasmer_serialized_module_t; + +/// Creates a new Module from the given wasm bytes. +/// +/// Returns `wasmer_result_t::WASMER_OK` upon success. +/// +/// Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` +/// and `wasmer_last_error_message` to get an error message. +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub unsafe extern "C" fn wasmer_compile( + module: *mut *mut wasmer_module_t, + wasm_bytes: *mut u8, + wasm_bytes_len: u32, +) -> wasmer_result_t { + let bytes: &[u8] = slice::from_raw_parts_mut(wasm_bytes, wasm_bytes_len as usize); + // TODO: this implicitly uses default_compiler() is that proper? a better way to handle metering? + let result = compile(bytes); + let new_module = match result { + Ok(instance) => instance, + Err(error) => { + update_last_error(error); + return wasmer_result_t::WASMER_ERROR; + } + }; + *module = Box::into_raw(Box::new(new_module)) as *mut wasmer_module_t; + wasmer_result_t::WASMER_OK +} + +/// Validates a sequence of bytes hoping it represents a valid WebAssembly module. +/// +/// The function returns true if the bytes are valid, false otherwise. +/// +/// Example: +/// +/// ```c +/// bool result = wasmer_validate(bytes, bytes_length); +/// +/// if (false == result) { +/// // Do something… +/// } +/// ``` +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub unsafe extern "C" fn wasmer_validate(wasm_bytes: *const u8, wasm_bytes_len: u32) -> bool { + if wasm_bytes.is_null() { + return false; + } + + let bytes: &[u8] = slice::from_raw_parts(wasm_bytes, wasm_bytes_len as usize); + + wasmer_runtime_core::validate(bytes) +} + +/// Creates a new Instance from the given module and imports. +/// +/// Returns `wasmer_result_t::WASMER_OK` upon success. +/// +/// Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` +/// and `wasmer_last_error_message` to get an error message. +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub unsafe extern "C" fn wasmer_module_instantiate( + module: *const wasmer_module_t, + instance: *mut *mut wasmer_instance_t, + imports: *mut wasmer_import_t, + imports_len: c_int, +) -> wasmer_result_t { + let imports: &[wasmer_import_t] = slice::from_raw_parts(imports, imports_len as usize); + let mut import_object = ImportObject::new(); + let mut namespaces = HashMap::new(); + for import in imports { + let module_name = slice::from_raw_parts( + import.module_name.bytes, + import.module_name.bytes_len as usize, + ); + let module_name = if let Ok(s) = std::str::from_utf8(module_name) { + s + } else { + update_last_error(CApiError { + msg: "error converting module name to string".to_string(), + }); + return wasmer_result_t::WASMER_ERROR; + }; + let import_name = slice::from_raw_parts( + import.import_name.bytes, + import.import_name.bytes_len as usize, + ); + let import_name = if let Ok(s) = std::str::from_utf8(import_name) { + s + } else { + update_last_error(CApiError { + msg: "error converting import_name to string".to_string(), + }); + return wasmer_result_t::WASMER_ERROR; + }; + + let namespace = namespaces.entry(module_name).or_insert_with(Namespace::new); + + let export = match import.tag { + wasmer_import_export_kind::WASM_MEMORY => { + let mem = import.value.memory as *mut Memory; + Export::Memory((&*mem).clone()) + } + wasmer_import_export_kind::WASM_FUNCTION => { + let func_export = import.value.func as *mut Export; + (&*func_export).clone() + } + wasmer_import_export_kind::WASM_GLOBAL => { + let global = import.value.global as *mut Global; + Export::Global((&*global).clone()) + } + wasmer_import_export_kind::WASM_TABLE => { + let table = import.value.table as *mut Table; + Export::Table((&*table).clone()) + } + }; + namespace.insert(import_name, export); + } + for (module_name, namespace) in namespaces.into_iter() { + import_object.register(module_name, namespace); + } + + let module = &*(module as *const Module); + let new_instance = match module.instantiate(&import_object) { + Ok(instance) => instance, + Err(error) => { + update_last_error(error); + return wasmer_result_t::WASMER_ERROR; + } + }; + + *instance = Box::into_raw(Box::new(new_instance)) as *mut wasmer_instance_t; + wasmer_result_t::WASMER_OK +} + +/// Given: +/// * A prepared `wasmer` import-object +/// * A compiled wasmer module +/// +/// Instantiates a wasmer instance +#[no_mangle] +pub unsafe extern "C" fn wasmer_module_import_instantiate( + instance: *mut *mut wasmer_instance_t, + module: *const wasmer_module_t, + import_object: *const wasmer_import_object_t, +) -> wasmer_result_t { + let import_object: &ImportObject = &*(import_object as *const ImportObject); + let module: &Module = &*(module as *const Module); + + let new_instance: Instance = match module.instantiate(import_object) { + Ok(instance) => instance, + Err(error) => { + update_last_error(error); + return wasmer_result_t::WASMER_ERROR; + } + }; + *instance = Box::into_raw(Box::new(new_instance)) as *mut wasmer_instance_t; + + return wasmer_result_t::WASMER_OK; +} + +/// Serialize the given Module. +/// +/// The caller owns the object and should call `wasmer_serialized_module_destroy` to free it. +/// +/// Returns `wasmer_result_t::WASMER_OK` upon success. +/// +/// Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` +/// and `wasmer_last_error_message` to get an error message. +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub unsafe extern "C" fn wasmer_module_serialize( + serialized_module: *mut *mut wasmer_serialized_module_t, + module: *const wasmer_module_t, +) -> wasmer_result_t { + let module = &*(module as *const Module); + + match module.cache() { + Ok(artifact) => match artifact.serialize() { + Ok(serialized_artifact) => { + *serialized_module = Box::into_raw(Box::new(serialized_artifact)) as _; + + wasmer_result_t::WASMER_OK + } + Err(_) => { + update_last_error(CApiError { + msg: "Failed to serialize the module artifact".to_string(), + }); + wasmer_result_t::WASMER_ERROR + } + }, + Err(_) => { + update_last_error(CApiError { + msg: "Failed to serialize the module".to_string(), + }); + wasmer_result_t::WASMER_ERROR + } + } +} + +/// Get bytes of the serialized module. +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub unsafe extern "C" fn wasmer_serialized_module_bytes( + serialized_module: *const wasmer_serialized_module_t, +) -> wasmer_byte_array { + let serialized_module = &*(serialized_module as *const &[u8]); + + wasmer_byte_array { + bytes: serialized_module.as_ptr(), + bytes_len: serialized_module.len() as u32, + } +} + +/// Transform a sequence of bytes into a serialized module. +/// +/// The caller owns the object and should call `wasmer_serialized_module_destroy` to free it. +/// +/// Returns `wasmer_result_t::WASMER_OK` upon success. +/// +/// Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` +/// and `wasmer_last_error_message` to get an error message. +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub unsafe extern "C" fn wasmer_serialized_module_from_bytes( + serialized_module: *mut *mut wasmer_serialized_module_t, + serialized_module_bytes: *const u8, + serialized_module_bytes_length: u32, +) -> wasmer_result_t { + if serialized_module.is_null() { + update_last_error(CApiError { + msg: "`serialized_module_bytes` pointer is null".to_string(), + }); + return wasmer_result_t::WASMER_ERROR; + } + + let serialized_module_bytes: &[u8] = slice::from_raw_parts( + serialized_module_bytes, + serialized_module_bytes_length as usize, + ); + + *serialized_module = Box::into_raw(Box::new(serialized_module_bytes)) as _; + wasmer_result_t::WASMER_OK +} + +/// Deserialize the given serialized module. +/// +/// Returns `wasmer_result_t::WASMER_OK` upon success. +/// +/// Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` +/// and `wasmer_last_error_message` to get an error message. +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub unsafe extern "C" fn wasmer_module_deserialize( + module: *mut *mut wasmer_module_t, + serialized_module: *const wasmer_serialized_module_t, +) -> wasmer_result_t { + if serialized_module.is_null() { + update_last_error(CApiError { + msg: "`serialized_module` pointer is null".to_string(), + }); + return wasmer_result_t::WASMER_ERROR; + } + + let serialized_module: &[u8] = &*(serialized_module as *const &[u8]); + + match Artifact::deserialize(serialized_module) { + // TODO: we need to use a different call here to support middleware (or modify wasmer-runtime) + Ok(artifact) => match load_cache_with(artifact, &default_compiler()) { + Ok(deserialized_module) => { + *module = Box::into_raw(Box::new(deserialized_module)) as _; + wasmer_result_t::WASMER_OK + } + Err(_) => { + update_last_error(CApiError { + msg: "Failed to compile the serialized module".to_string(), + }); + wasmer_result_t::WASMER_ERROR + } + }, + Err(_) => { + update_last_error(CApiError { + msg: "Failed to deserialize the module".to_string(), + }); + wasmer_result_t::WASMER_ERROR + } + } +} + +/// Frees memory for the given serialized Module. +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub extern "C" fn wasmer_serialized_module_destroy( + serialized_module: *mut wasmer_serialized_module_t, +) { + if !serialized_module.is_null() { + unsafe { Box::from_raw(serialized_module as *mut &[u8]) }; + } +} + +/// Frees memory for the given Module +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub extern "C" fn wasmer_module_destroy(module: *mut wasmer_module_t) { + if !module.is_null() { + unsafe { Box::from_raw(module as *mut Module) }; + } +} diff --git a/lib/runtime-c-api/src/runtime_breakpoints.rs b/lib/runtime-c-api/src/runtime_breakpoints.rs new file mode 100644 index 000000000000..dc64118963a1 --- /dev/null +++ b/lib/runtime-c-api/src/runtime_breakpoints.rs @@ -0,0 +1,33 @@ +use crate::instance::wasmer_instance_t; + +use wasmer_middleware_common::runtime_breakpoints::{ + set_runtime_breakpoint_value, + get_runtime_breakpoint_value, + BREAKPOINT_VALUE_NO_BREAKPOINT +}; + +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub unsafe extern "C" fn wasmer_instance_set_runtime_breakpoint_value( + instance: *mut wasmer_instance_t, + value: u64, +) { + if instance.is_null() { + return; + } + let instance = &mut *(instance as *mut wasmer_runtime::Instance); + set_runtime_breakpoint_value(instance, value); +} + +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub unsafe extern "C" fn wasmer_instance_get_runtime_breakpoint_value( + instance: *mut wasmer_instance_t, +) -> u64 { + if instance.is_null() { + return BREAKPOINT_VALUE_NO_BREAKPOINT; + } + let instance = &mut *(instance as *mut wasmer_runtime::Instance); + + get_runtime_breakpoint_value(instance) +} diff --git a/lib/runtime-c-api/src/signals.rs b/lib/runtime-c-api/src/signals.rs new file mode 100644 index 000000000000..9feba9f9dcde --- /dev/null +++ b/lib/runtime-c-api/src/signals.rs @@ -0,0 +1,8 @@ +use wasmer_runtime_core::fault::SIGSEGV_PASSTHROUGH; +use std::sync::atomic::Ordering; + +#[allow(clippy::cast_ptr_alignment)] +#[no_mangle] +pub unsafe extern "C" fn wasmer_set_sigsegv_passthrough() { + SIGSEGV_PASSTHROUGH.swap(true, Ordering::SeqCst); +} diff --git a/lib/runtime-c-api/tests/.gitignore b/lib/runtime-c-api/tests/.gitignore new file mode 100644 index 000000000000..813e3efd7066 --- /dev/null +++ b/lib/runtime-c-api/tests/.gitignore @@ -0,0 +1,32 @@ +CMakeLists.txt.user +CMakeCache.txt +CMakeFiles +CMakeScripts +Testing +Makefile +cmake_install.cmake +install_manifest.txt +compile_commands.json +CTestTestfile.cmake +_deps +rust-build +test-context +test-exported-memory +test-exports +test-globals +test-import-function +test-import-trap +test-import-object +test-imports +test-instantiate +test-memory +test-module +test-module-exports +test-module-import-instantiate +test-module-imports +test-module-serialize +test-module-metering-serialize +test-tables +test-validate +test-wasi-import-object +test-emscripten-import-object \ No newline at end of file diff --git a/lib/runtime-c-api/tests/CMakeLists.txt b/lib/runtime-c-api/tests/CMakeLists.txt new file mode 100644 index 000000000000..2a741f2732b8 --- /dev/null +++ b/lib/runtime-c-api/tests/CMakeLists.txt @@ -0,0 +1,136 @@ +cmake_minimum_required (VERSION 2.6) +project (WasmerRuntimeCApiTests) + +add_executable(test-exported-memory test-exported-memory.c) +add_executable(test-exports test-exports.c) +add_executable(test-globals test-globals.c) +add_executable(test-import-function test-import-function.c) +add_executable(test-import-trap test-import-trap.c) +add_executable(test-imports test-imports.c) +add_executable(test-import-object test-import-object.c) +add_executable(test-instantiate test-instantiate.c) +add_executable(test-memory test-memory.c) +add_executable(test-module test-module.c) +add_executable(test-module-exports test-module-exports.c) +add_executable(test-module-imports test-module-imports.c) +add_executable(test-module-serialize test-module-serialize.c) +add_executable(test-module-metering-serialize test-module-metering-serialize.c) +add_executable(test-tables test-tables.c) +add_executable(test-validate test-validate.c) +add_executable(test-context test-context.c) +add_executable(test-module-import-instantiate test-module-import-instantiate.c) + +if (DEFINED WASI_TESTS) + add_executable(test-wasi-import-object test-wasi-import-object.c) +endif() + +if (DEFINED EMSCRIPTEN_TESTS) + add_executable(test-emscripten-import-object test-emscripten-import-object.c) +endif() + + +find_library( + WASMER_LIB NAMES libwasmer_runtime_c_api.dylib libwasmer_runtime_c_api.so wasmer_runtime_c_api.dll + PATHS ${CMAKE_SOURCE_DIR}/../../../target/release/ +) + +if(NOT WASMER_LIB) + message(FATAL_ERROR "wasmer library not found") +endif() + +enable_testing() + +set( + COMPILER_OPTIONS + # Clang or gcc + $<$,$>: + "-Werror" > + # MSVC + $<$: + "/WX" > +) + +target_link_libraries(test-exported-memory general ${WASMER_LIB}) +target_compile_options(test-exported-memory PRIVATE ${COMPILER_OPTIONS}) +add_test(test-exported-memory test-exported-memory) + +target_link_libraries(test-exports general ${WASMER_LIB}) +target_compile_options(test-exports PRIVATE ${COMPILER_OPTIONS}) +add_test(test-exports test-exports) + +target_link_libraries(test-globals general ${WASMER_LIB}) +target_compile_options(test-globals PRIVATE ${COMPILER_OPTIONS}) +add_test(test-globals test-globals) + +target_link_libraries(test-import-function general ${WASMER_LIB}) +target_compile_options(test-import-function PRIVATE ${COMPILER_OPTIONS}) +add_test(test-import-function test-import-function) + +target_link_libraries(test-import-trap general ${WASMER_LIB}) +target_compile_options(test-import-trap PRIVATE ${COMPILER_OPTIONS}) +add_test(test-import-trap test-import-trap) + +target_link_libraries(test-imports general ${WASMER_LIB}) +target_compile_options(test-imports PRIVATE ${COMPILER_OPTIONS}) +add_test(test-imports test-imports) + +target_link_libraries(test-import-object general ${WASMER_LIB}) +target_compile_options(test-import-object PRIVATE ${COMPILER_OPTIONS}) +add_test(test-import-object test-import-object) + + +if (DEFINED WASI_TESTS) + target_link_libraries(test-wasi-import-object general ${WASMER_LIB}) + target_compile_options(test-wasi-import-object PRIVATE ${COMPILER_OPTIONS}) + add_test(test-wasi-import-object test-wasi-import-object) +endif() + +if (DEFINED EMSCRIPTEN_TESTS) + target_link_libraries(test-emscripten-import-object general ${WASMER_LIB}) + target_compile_options(test-emscripten-import-object PRIVATE ${COMPILER_OPTIONS}) + add_test(test-emscripten-import-object test-emscripten-import-object) +endif() + +target_link_libraries(test-instantiate general ${WASMER_LIB}) +target_compile_options(test-instantiate PRIVATE ${COMPILER_OPTIONS}) +add_test(test-instantiate test-instantiate) + +target_link_libraries(test-memory general ${WASMER_LIB}) +target_compile_options(test-memory PRIVATE ${COMPILER_OPTIONS}) +add_test(test-memory test-memory) + +target_link_libraries(test-module general ${WASMER_LIB}) +target_compile_options(test-module PRIVATE ${COMPILER_OPTIONS}) +add_test(test-module test-module) + +target_link_libraries(test-module-exports general ${WASMER_LIB}) +target_compile_options(test-module-exports PRIVATE ${COMPILER_OPTIONS}) +add_test(test-module-exports test-module-exports) + +target_link_libraries(test-module-imports general ${WASMER_LIB}) +target_compile_options(test-module-imports PRIVATE ${COMPILER_OPTIONS}) +add_test(test-module-imports test-module-imports) + +target_link_libraries(test-module-serialize general ${WASMER_LIB}) +target_compile_options(test-module-serialize PRIVATE ${COMPILER_OPTIONS}) +add_test(test-module-serialize test-module-serialize) + +target_link_libraries(test-module-metering-serialize general ${WASMER_LIB}) +target_compile_options(test-module-metering-serialize PRIVATE ${COMPILER_OPTIONS}) +add_test(test-module-metering-serialize test-module-metering-serialize) + +target_link_libraries(test-tables general ${WASMER_LIB}) +target_compile_options(test-tables PRIVATE ${COMPILER_OPTIONS}) +add_test(test-tables test-tables) + +target_link_libraries(test-validate general ${WASMER_LIB}) +target_compile_options(test-validate PRIVATE ${COMPILER_OPTIONS}) +add_test(test-validate test-validate) + +target_link_libraries(test-context general ${WASMER_LIB}) +target_compile_options(test-context PRIVATE ${COMPILER_OPTIONS}) +add_test(test-context test-context) + +target_link_libraries(test-module-import-instantiate general ${WASMER_LIB}) +target_compile_options(test-module-import-instantiate PRIVATE ${COMPILER_OPTIONS}) +add_test(test-module-import-instantiate test-module-import-instantiate) diff --git a/lib/runtime-c-api/tests/test-module-metering-serialize.c b/lib/runtime-c-api/tests/test-module-metering-serialize.c new file mode 100644 index 000000000000..4b30f90558c2 --- /dev/null +++ b/lib/runtime-c-api/tests/test-module-metering-serialize.c @@ -0,0 +1,131 @@ +#include +#include "../wasmer.h" +#include +#include + +int main() +{ + // Read the wasm file bytes + FILE *file = fopen("assets/sum.wasm", "r"); + fseek(file, 0, SEEK_END); + long len = ftell(file); + uint8_t *bytes = malloc(len); + fseek(file, 0, SEEK_SET); + fread(bytes, 1, len, file); + fclose(file); + + wasmer_module_t *module_one = NULL; + unsigned long long gas_limit = 100; + wasmer_result_t compile_result = wasmer_compile_with_gas_metering(&module_one, bytes, len); + printf("Compile result: %d\n", compile_result); + assert(compile_result == WASMER_OK); + + // first run before serialization + wasmer_import_t imports[] = {}; + wasmer_instance_t *instance_one = NULL; + wasmer_result_t instantiate_result = wasmer_module_instantiate(module_one, &instance_one, imports, 0); + printf("Instantiate result: %d\n", instantiate_result); + assert(instantiate_result == WASMER_OK); + + // check behavior of getting/setting points, also no error on null + assert(wasmer_instance_get_points_used(instance_one) == 0); + wasmer_instance_set_points_used(instance_one, 50); + assert(wasmer_instance_get_points_used(instance_one) == 50); + assert(wasmer_instance_get_points_used(NULL) == 0); + + wasmer_value_t param_one; + param_one.tag = WASM_I32; + param_one.value.I32 = 7; + wasmer_value_t param_two; + param_two.tag = WASM_I32; + param_two.value.I32 = 8; + wasmer_value_t params[] = {param_one, param_two}; + + wasmer_value_t result_one; + wasmer_value_t results[] = {result_one}; + + wasmer_result_t call_result = wasmer_instance_call(instance_one, "sum", params, 2, results, 1); + printf("Call result: %d\n", call_result); + printf("Result: %d\n", results[0].value.I32); + assert(results[0].value.I32 == 15); + assert(call_result == WASMER_OK); + + // ensure we got charged some gas + assert(wasmer_instance_get_points_used(instance_one) == 54); + // TODO: try again an ensure limit enforced... need another function + + // end first run + + wasmer_serialized_module_t *serialized_module = NULL; + wasmer_result_t serialize_result = wasmer_module_serialize(&serialized_module, module_one); + printf("Serialize result: %d\n", serialize_result); + assert(serialize_result == WASMER_OK); + + wasmer_byte_array serialized_module_bytes = wasmer_serialized_module_bytes(serialized_module); + printf("Serialized module pointer: %p\n", serialized_module_bytes.bytes); + printf("Serialized module length: %d\n", serialized_module_bytes.bytes_len); + assert(serialized_module_bytes.bytes != NULL); + assert(serialized_module_bytes.bytes_len > 8); + assert(serialized_module_bytes.bytes[0] == 'W'); + assert(serialized_module_bytes.bytes[1] == 'A'); + assert(serialized_module_bytes.bytes[2] == 'S'); + assert(serialized_module_bytes.bytes[3] == 'M'); + assert(serialized_module_bytes.bytes[4] == 'E'); + assert(serialized_module_bytes.bytes[5] == 'R'); + + wasmer_module_t *module_two = NULL; + wasmer_result_t unserialize_result = wasmer_module_deserialize(&module_two, serialized_module); + assert(unserialize_result == WASMER_OK); + + // second run with deserialized module + wasmer_instance_t *instance_two = NULL; + instantiate_result = wasmer_module_instantiate(module_two, &instance_two, imports, 0); + printf("Instantiate result: %d\n", instantiate_result); + assert(instantiate_result == WASMER_OK); + + // ensure points independent of other instance + assert(wasmer_instance_get_points_used(instance_one) > 50); + assert(wasmer_instance_get_points_used(instance_two) == 0); + wasmer_instance_set_points_used(instance_two, 20); + assert(wasmer_instance_get_points_used(instance_one) > 50); + assert(wasmer_instance_get_points_used(instance_two) == 20); + + call_result = wasmer_instance_call(instance_two, "sum", params, 2, results, 1); + printf("Call result: %d\n", call_result); + printf("Result: %d\n", results[0].value.I32); + assert(results[0].value.I32 == 15); + assert(call_result == WASMER_OK); + + // and we charge the right one + assert(wasmer_instance_get_points_used(instance_two) == 24); + + wasmer_serialized_module_t *serialized_module_two = NULL; + wasmer_result_t serialized_module_from_bytes_result = wasmer_serialized_module_from_bytes( + &serialized_module_two, + serialized_module_bytes.bytes, + serialized_module_bytes.bytes_len + ); + assert(serialized_module_from_bytes_result == WASMER_OK); + + wasmer_module_t *module_three = NULL; + wasmer_result_t unserialized_result_two = wasmer_module_deserialize(&module_three, serialized_module_two); + assert(unserialized_result_two == WASMER_OK); + + wasmer_instance_t *instance_three = NULL; + wasmer_result_t instantiate_result_two = wasmer_module_instantiate(module_three, &instance_three, imports, 0); + assert(instantiate_result_two == WASMER_OK); + + printf("Destroy the serialized module\n"); + wasmer_serialized_module_destroy(serialized_module); + wasmer_serialized_module_destroy(serialized_module_two); + + printf("Destroy instance\n"); + wasmer_instance_destroy(instance_one); + wasmer_instance_destroy(instance_two); + wasmer_instance_destroy(instance_three); + + printf("Destroy modules\n"); + wasmer_module_destroy(module_one); + wasmer_module_destroy(module_two); + return 0; +} diff --git a/lib/runtime-c-api/wasmer.h b/lib/runtime-c-api/wasmer.h new file mode 100644 index 000000000000..cbcf4b60ce77 --- /dev/null +++ b/lib/runtime-c-api/wasmer.h @@ -0,0 +1,1595 @@ + +#if !defined(WASMER_H_MACROS) + +#define WASMER_H_MACROS + +// Define the `ARCH_X86_X64` constant. +#if defined(MSVC) && defined(_M_AMD64) +# define ARCH_X86_64 +#elif (defined(GCC) || defined(__GNUC__) || defined(__clang__)) && defined(__x86_64__) +# define ARCH_X86_64 +#endif + +// Compatibility with non-Clang compilers. +#if !defined(__has_attribute) +# define __has_attribute(x) 0 +#endif + +// Compatibility with non-Clang compilers. +#if !defined(__has_declspec_attribute) +# define __has_declspec_attribute(x) 0 +#endif + +// Define the `DEPRECATED` macro. +#if defined(GCC) || defined(__GNUC__) || __has_attribute(deprecated) +# define DEPRECATED(message) __attribute__((deprecated(message))) +#elif defined(MSVC) || __has_declspec_attribute(deprecated) +# define DEPRECATED(message) __declspec(deprecated(message)) +#endif + +#endif // WASMER_H_MACROS + + +#ifndef WASMER_H +#define WASMER_H + +#include +#include +#include +#include + +#define OPCODE_COUNT 448 + +#if defined(WASMER_WASI_ENABLED) +enum Version { + /** + * Version cannot be detected or is unknown. + */ + Unknown = 0, + /** + * Latest version. See `wasmer_wasi::WasiVersion::Latest` to + * learn more. + */ + Latest = 1, + /** + * `wasi_unstable`. + */ + Snapshot0 = 2, + /** + * `wasi_snapshot_preview1`. + */ + Snapshot1 = 3, +}; +typedef uint8_t Version; +#endif + +/** + * List of export/import kinds. + */ +enum wasmer_import_export_kind { + /** + * The export/import is a function. + */ + WASM_FUNCTION = 0, + /** + * The export/import is a global. + */ + WASM_GLOBAL = 1, + /** + * The export/import is a memory. + */ + WASM_MEMORY = 2, + /** + * The export/import is a table. + */ + WASM_TABLE = 3, +}; +typedef uint32_t wasmer_import_export_kind; + +/** + * The `wasmer_result_t` enum is a type that represents either a + * success, or a failure. + */ +typedef enum { + /** + * Represents a success. + */ + WASMER_OK = 1, + /** + * Represents a failure. + */ + WASMER_ERROR = 2, +} wasmer_result_t; + +/** + * Represents all possibles WebAssembly value types. + * + * See `wasmer_value_t` to get a complete example. + */ +enum wasmer_value_tag { + /** + * Represents the `i32` WebAssembly type. + */ + WASM_I32, + /** + * Represents the `i64` WebAssembly type. + */ + WASM_I64, + /** + * Represents the `f32` WebAssembly type. + */ + WASM_F32, + /** + * Represents the `f64` WebAssembly type. + */ + WASM_F64, +}; +typedef uint32_t wasmer_value_tag; + +typedef struct { + +} wasmer_module_t; + +/** + * Opaque pointer to a `wasmer_runtime::Instance` value in Rust. + * + * A `wasmer_runtime::Instance` represents a WebAssembly instance. It + * is generally generated by the `wasmer_instantiate()` function, or by + * the `wasmer_module_instantiate()` function for the most common paths. + */ +typedef struct { + +} wasmer_instance_t; + +typedef struct { + const uint8_t *bytes; + uint32_t bytes_len; +} wasmer_byte_array; + +#if defined(WASMER_EMSCRIPTEN_ENABLED) +/** + * Type used to construct an import_object_t with Emscripten imports. + */ +typedef struct { + +} wasmer_emscripten_globals_t; +#endif + +typedef struct { + +} wasmer_import_object_t; + +/** + * Opaque pointer to `NamedExportDescriptor`. + */ +typedef struct { + +} wasmer_export_descriptor_t; + +/** + * Opaque pointer to `NamedExportDescriptors`. + */ +typedef struct { + +} wasmer_export_descriptors_t; + +/** + * Opaque pointer to `wasmer_export_t`. + */ +typedef struct { + +} wasmer_export_func_t; + +/** + * Represents a WebAssembly value. + * + * This is a [Rust union][rust-union], which is equivalent to the C + * union. See `wasmer_value_t` to get a complete example. + * + * [rust-union]: https://doc.rust-lang.org/reference/items/unions.html + */ +typedef union { + int32_t I32; + int64_t I64; + float F32; + double F64; +} wasmer_value; + +/** + * Represents a WebAssembly type and value pair, + * i.e. `wasmer_value_tag` and `wasmer_value`. Since the latter is an + * union, it's the safe way to read or write a WebAssembly value in + * C. + * + * Example: + * + * ```c + * // Create a WebAssembly value. + * wasmer_value_t wasm_value = { + * .tag = WASM_I32, + * .value.I32 = 42, + * }; + * + * // Read a WebAssembly value. + * if (wasm_value.tag == WASM_I32) { + * int32_t x = wasm_value.value.I32; + * // … + * } + * ``` + */ +typedef struct { + /** + * The value type. + */ + wasmer_value_tag tag; + /** + * The value. + */ + wasmer_value value; +} wasmer_value_t; + +/** + * Opaque pointer to `NamedExport`. + */ +typedef struct { + +} wasmer_export_t; + +/** + * Opaque pointer to a `wasmer_runtime::Memory` value in Rust. + * + * A `wasmer_runtime::Memory` represents a WebAssembly memory. It is + * possible to create one with `wasmer_memory_new()` and pass it as + * imports of an instance, or to read it from exports of an instance + * with `wasmer_export_to_memory()`. + */ +typedef struct { + +} wasmer_memory_t; + +/** + * Opaque pointer to the opaque structure `crate::NamedExports`, + * which is a wrapper around a vector of the opaque structure + * `crate::NamedExport`. + * + * Check the `wasmer_instance_exports()` function to learn more. + */ +typedef struct { + +} wasmer_exports_t; + +typedef struct { + +} wasmer_global_t; + +typedef struct { + bool mutable_; + wasmer_value_tag kind; +} wasmer_global_descriptor_t; + +typedef struct { + +} wasmer_import_descriptor_t; + +typedef struct { + +} wasmer_import_descriptors_t; + +typedef struct { + +} wasmer_import_func_t; + +typedef struct { + +} wasmer_table_t; + +/** + * Union of import/export value. + */ +typedef union { + const wasmer_import_func_t *func; + const wasmer_table_t *table; + const wasmer_memory_t *memory; + const wasmer_global_t *global; +} wasmer_import_export_value; + +typedef struct { + wasmer_byte_array module_name; + wasmer_byte_array import_name; + wasmer_import_export_kind tag; + wasmer_import_export_value value; +} wasmer_import_t; + +typedef struct { + +} wasmer_import_object_iter_t; + +/** + * Opaque pointer to a `wasmer_runtime::Ctx` value in Rust. + * + * An instance context is passed to any host function (aka imported + * function) as the first argument. It is necessary to read the + * instance data or the memory, respectively with the + * `wasmer_instance_context_data_get()` function, and the + * `wasmer_instance_context_memory()` function. + * + * It is also possible to get the instance context outside a host + * function by using the `wasmer_instance_context_get()` + * function. See also `wasmer_instance_context_data_set()` to set the + * instance context data. + * + * Example: + * + * ```c + * // A host function that prints data from the WebAssembly memory to + * // the standard output. + * void print(wasmer_instance_context_t *context, int32_t pointer, int32_t length) { + * // Use `wasmer_instance_context` to get back the first instance memory. + * const wasmer_memory_t *memory = wasmer_instance_context_memory(context, 0); + * + * // Continue… + * } + * ``` + */ +typedef struct { + +} wasmer_instance_context_t; + +typedef struct { + +} wasmer_compilation_options_t; + +/** + * The `wasmer_limit_option_t` struct represents an optional limit + * for `wasmer_limits_t`. + */ +typedef struct { + /** + * Whether the limit is set. + */ + bool has_some; + /** + * The limit value. + */ + uint32_t some; +} wasmer_limit_option_t; + +/** + * The `wasmer_limits_t` struct is a type that describes a memory + * options. See the `wasmer_memory_t` struct or the + * `wasmer_memory_new()` function to get more information. + */ +typedef struct { + /** + * The minimum number of allowed pages. + */ + uint32_t min; + /** + * The maximum number of allowed pages. + */ + wasmer_limit_option_t max; +} wasmer_limits_t; + +typedef struct { + +} wasmer_serialized_module_t; + +#if (!defined(_WIN32) && defined(ARCH_X86_64)) +typedef struct { + +} wasmer_trampoline_buffer_builder_t; +#endif + +#if (!defined(_WIN32) && defined(ARCH_X86_64)) +typedef struct { + +} wasmer_trampoline_callable_t; +#endif + +#if (!defined(_WIN32) && defined(ARCH_X86_64)) +typedef struct { + +} wasmer_trampoline_buffer_t; +#endif + +#if defined(WASMER_WASI_ENABLED) +/** + * Opens a directory that's visible to the WASI module as `alias` but + * is backed by the host file at `host_file_path` + */ +typedef struct { + /** + * What the WASI module will see in its virtual root + */ + wasmer_byte_array alias; + /** + * The backing file that the WASI module will interact with via the alias + */ + wasmer_byte_array host_file_path; +} wasmer_wasi_map_dir_entry_t; +#endif + +/** + * Creates a new Module from the given wasm bytes. + * + * Returns `wasmer_result_t::WASMER_OK` upon success. + * + * Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` + * and `wasmer_last_error_message` to get an error message. + */ +wasmer_result_t wasmer_compile(wasmer_module_t **module, + uint8_t *wasm_bytes, + uint32_t wasm_bytes_len); + +/** + * Creates a new Module with gas limit from the given wasm bytes. + * + * Returns `wasmer_result_t::WASMER_OK` upon success. + * + * Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` + * and `wasmer_last_error_message` to get an error message. + */ +wasmer_result_t wasmer_compile_with_gas_metering(wasmer_module_t **module, + uint8_t *wasm_bytes, + uint32_t wasm_bytes_len); + +wasmer_result_t wasmer_compile_with_gas_metering(wasmer_module_t **module, + uint8_t *wasm_bytes, + uint32_t wasm_bytes_len); + +#if defined(WASMER_EMSCRIPTEN_ENABLED) +/** + * Convenience function for setting up arguments and calling the Emscripten + * main function. + * + * WARNING: + * + * Do not call this function on untrusted code when operating without + * additional sandboxing in place. + * Emscripten has access to many host system calls and therefore may do very + * bad things. + */ +wasmer_result_t wasmer_emscripten_call_main(wasmer_instance_t *instance, + const wasmer_byte_array *args, + unsigned int args_len); +#endif + +#if defined(WASMER_EMSCRIPTEN_ENABLED) +/** + * Destroy `wasmer_emscrpten_globals_t` created by + * `wasmer_emscripten_get_emscripten_globals`. + */ +void wasmer_emscripten_destroy_globals(wasmer_emscripten_globals_t *globals); +#endif + +#if defined(WASMER_EMSCRIPTEN_ENABLED) +/** + * Create a `wasmer_import_object_t` with Emscripten imports, use + * `wasmer_emscripten_get_emscripten_globals` to get a + * `wasmer_emscripten_globals_t` from a `wasmer_module_t`. + * + * WARNING: + * + * This `import_object_t` contains thin-wrappers around host system calls. + * Do not use this to execute untrusted code without additional sandboxing. + */ +wasmer_import_object_t *wasmer_emscripten_generate_import_object(wasmer_emscripten_globals_t *globals); +#endif + +#if defined(WASMER_EMSCRIPTEN_ENABLED) +/** + * Create a `wasmer_emscripten_globals_t` from a Wasm module. + */ +wasmer_emscripten_globals_t *wasmer_emscripten_get_globals(const wasmer_module_t *module); +#endif + +#if defined(WASMER_EMSCRIPTEN_ENABLED) +/** + * Execute global constructors (required if the module is compiled from C++) + * and sets up the internal environment. + * + * This function sets the data pointer in the same way that + * [`wasmer_instance_context_data_set`] does. + */ +wasmer_result_t wasmer_emscripten_set_up(wasmer_instance_t *instance, + wasmer_emscripten_globals_t *globals); +#endif + +/** + * Gets export descriptor kind + */ +wasmer_import_export_kind wasmer_export_descriptor_kind(wasmer_export_descriptor_t *export_); + +/** + * Gets name for the export descriptor + */ +wasmer_byte_array wasmer_export_descriptor_name(wasmer_export_descriptor_t *export_descriptor); + +/** + * Gets export descriptors for the given module + * + * The caller owns the object and should call `wasmer_export_descriptors_destroy` to free it. + */ +void wasmer_export_descriptors(const wasmer_module_t *module, + wasmer_export_descriptors_t **export_descriptors); + +/** + * Frees the memory for the given export descriptors + */ +void wasmer_export_descriptors_destroy(wasmer_export_descriptors_t *export_descriptors); + +/** + * Gets export descriptor by index + */ +wasmer_export_descriptor_t *wasmer_export_descriptors_get(wasmer_export_descriptors_t *export_descriptors, + int idx); + +/** + * Gets the length of the export descriptors + */ +int wasmer_export_descriptors_len(wasmer_export_descriptors_t *exports); + +/** + * Calls a `func` with the provided parameters. + * Results are set using the provided `results` pointer. + * + * Returns `wasmer_result_t::WASMER_OK` upon success. + * + * Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` + * and `wasmer_last_error_message` to get an error message. + */ +wasmer_result_t wasmer_export_func_call(const wasmer_export_func_t *func, + const wasmer_value_t *params, + unsigned int params_len, + wasmer_value_t *results, + unsigned int results_len); + +/** + * Sets the params buffer to the parameter types of the given wasmer_export_func_t + * + * Returns `wasmer_result_t::WASMER_OK` upon success. + * + * Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` + * and `wasmer_last_error_message` to get an error message. + */ +wasmer_result_t wasmer_export_func_params(const wasmer_export_func_t *func, + wasmer_value_tag *params, + uint32_t params_len); + +/** + * Sets the result parameter to the arity of the params of the wasmer_export_func_t + * + * Returns `wasmer_result_t::WASMER_OK` upon success. + * + * Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` + * and `wasmer_last_error_message` to get an error message. + */ +wasmer_result_t wasmer_export_func_params_arity(const wasmer_export_func_t *func, uint32_t *result); + +/** + * Sets the returns buffer to the parameter types of the given wasmer_export_func_t + * + * Returns `wasmer_result_t::WASMER_OK` upon success. + * + * Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` + * and `wasmer_last_error_message` to get an error message. + */ +wasmer_result_t wasmer_export_func_returns(const wasmer_export_func_t *func, + wasmer_value_tag *returns, + uint32_t returns_len); + +/** + * Sets the result parameter to the arity of the returns of the wasmer_export_func_t + * + * Returns `wasmer_result_t::WASMER_OK` upon success. + * + * Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` + * and `wasmer_last_error_message` to get an error message. + */ +wasmer_result_t wasmer_export_func_returns_arity(const wasmer_export_func_t *func, + uint32_t *result); + +/** + * Gets wasmer_export kind + */ +wasmer_import_export_kind wasmer_export_kind(wasmer_export_t *export_); + +/** + * Gets name from wasmer_export + */ +wasmer_byte_array wasmer_export_name(wasmer_export_t *export_); + +/** + * Gets export func from export + */ +const wasmer_export_func_t *wasmer_export_to_func(const wasmer_export_t *export_); + +/** + * Gets a memory pointer from an export pointer. + * + * Returns `wasmer_result_t::WASMER_OK` upon success. + * + * Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` + * and `wasmer_last_error_message` to get an error message. + */ +wasmer_result_t wasmer_export_to_memory(const wasmer_export_t *export_, wasmer_memory_t **memory); + +/** + * Frees the memory for the given exports. + * + * Check the `wasmer_instance_exports()` function to get a complete + * example. + * + * If `exports` is a null pointer, this function does nothing. + * + * Example: + * + * ```c + * // Get some exports. + * wasmer_exports_t *exports = NULL; + * wasmer_instance_exports(instance, &exports); + * + * // Destroy the exports. + * wasmer_exports_destroy(exports); + * ``` + */ +void wasmer_exports_destroy(wasmer_exports_t *exports); + +/** + * Gets wasmer_export by index + */ +wasmer_export_t *wasmer_exports_get(wasmer_exports_t *exports, int idx); + +/** + * Gets the length of the exports + */ +int wasmer_exports_len(wasmer_exports_t *exports); + +/** + * Frees memory for the given Global + */ +void wasmer_global_destroy(wasmer_global_t *global); + +/** + * Gets the value stored by the given Global + */ +wasmer_value_t wasmer_global_get(wasmer_global_t *global); + +/** + * Returns a descriptor (type, mutability) of the given Global + */ +wasmer_global_descriptor_t wasmer_global_get_descriptor(wasmer_global_t *global); + +/** + * Creates a new Global and returns a pointer to it. + * The caller owns the object and should call `wasmer_global_destroy` to free it. + */ +wasmer_global_t *wasmer_global_new(wasmer_value_t value, bool mutable_); + +/** + * Sets the value stored by the given Global + */ +void wasmer_global_set(wasmer_global_t *global, wasmer_value_t value); + +/** + * Gets export descriptor kind + */ +wasmer_import_export_kind wasmer_import_descriptor_kind(wasmer_import_descriptor_t *export_); + +/** + * Gets module name for the import descriptor + */ +wasmer_byte_array wasmer_import_descriptor_module_name(wasmer_import_descriptor_t *import_descriptor); + +/** + * Gets name for the import descriptor + */ +wasmer_byte_array wasmer_import_descriptor_name(wasmer_import_descriptor_t *import_descriptor); + +/** + * Gets import descriptors for the given module + * + * The caller owns the object and should call `wasmer_import_descriptors_destroy` to free it. + */ +void wasmer_import_descriptors(const wasmer_module_t *module, + wasmer_import_descriptors_t **import_descriptors); + +/** + * Frees the memory for the given import descriptors + */ +void wasmer_import_descriptors_destroy(wasmer_import_descriptors_t *import_descriptors); + +/** + * Gets import descriptor by index + */ +wasmer_import_descriptor_t *wasmer_import_descriptors_get(wasmer_import_descriptors_t *import_descriptors, + unsigned int idx); + +/** + * Gets the length of the import descriptors + */ +unsigned int wasmer_import_descriptors_len(wasmer_import_descriptors_t *exports); + +/** + * Frees memory for the given Func + */ +void wasmer_import_func_destroy(wasmer_import_func_t *func); + +/** + * Creates new host function, aka imported function. `func` is a + * function pointer, where the first argument is the famous `vm::Ctx` + * (in Rust), or `wasmer_instance_context_t` (in C). All arguments + * must be typed with compatible WebAssembly native types: + * + * | WebAssembly type | C/C++ type | + * | ---------------- | ---------- | + * | `i32` | `int32_t` | + * | `i64` | `int64_t` | + * | `f32` | `float` | + * | `f64` | `double` | + * + * The function pointer must have a lifetime greater than the + * WebAssembly instance lifetime. + * + * The caller owns the object and should call + * `wasmer_import_func_destroy` to free it. + */ +wasmer_import_func_t *wasmer_import_func_new(void (*func)(void *data), + const wasmer_value_tag *params, + unsigned int params_len, + const wasmer_value_tag *returns, + unsigned int returns_len); + +/** + * Sets the params buffer to the parameter types of the given wasmer_import_func_t + * + * Returns `wasmer_result_t::WASMER_OK` upon success. + * + * Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` + * and `wasmer_last_error_message` to get an error message. + */ +wasmer_result_t wasmer_import_func_params(const wasmer_import_func_t *func, + wasmer_value_tag *params, + unsigned int params_len); + +/** + * Sets the result parameter to the arity of the params of the wasmer_import_func_t + * + * Returns `wasmer_result_t::WASMER_OK` upon success. + * + * Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` + * and `wasmer_last_error_message` to get an error message. + */ +wasmer_result_t wasmer_import_func_params_arity(const wasmer_import_func_t *func, uint32_t *result); + +/** + * Sets the returns buffer to the parameter types of the given wasmer_import_func_t + * + * Returns `wasmer_result_t::WASMER_OK` upon success. + * + * Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` + * and `wasmer_last_error_message` to get an error message. + */ +wasmer_result_t wasmer_import_func_returns(const wasmer_import_func_t *func, + wasmer_value_tag *returns, + unsigned int returns_len); + +/** + * Sets the result parameter to the arity of the returns of the wasmer_import_func_t + * + * Returns `wasmer_result_t::WASMER_OK` upon success. + * + * Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` + * and `wasmer_last_error_message` to get an error message. + */ +wasmer_result_t wasmer_import_func_returns_arity(const wasmer_import_func_t *func, + uint32_t *result); + +wasmer_result_t wasmer_import_object_cache_from_imports(wasmer_import_t *imports, + unsigned int imports_len); + +/** + * Frees memory of the given ImportObject + */ +void wasmer_import_object_destroy(wasmer_import_object_t *import_object); + +/** + * Extends an existing import object with new imports + */ +wasmer_result_t wasmer_import_object_extend(wasmer_import_object_t *import_object, + const wasmer_import_t *imports, + unsigned int imports_len); + +/** + * Gets an entry from an ImportObject at the name and namespace. + * Stores `name`, `namespace`, and `import_export_value` in `import`. + * Thus these must remain valid for the lifetime of `import`. + * + * The caller owns all data involved. + * `import_export_value` will be written to based on `tag`. + */ +wasmer_result_t wasmer_import_object_get_import(const wasmer_import_object_t *import_object, + wasmer_byte_array namespace_, + wasmer_byte_array name, + wasmer_import_t *import, + wasmer_import_export_value *import_export_value, + uint32_t tag); + +/** + * Frees the memory allocated in `wasmer_import_object_iter_next` + * + * This function does not free the memory in `wasmer_import_object_t`; + * it only frees memory allocated while querying a `wasmer_import_object_t`. + */ +void wasmer_import_object_imports_destroy(wasmer_import_t *imports, uint32_t imports_len); + +/** + * Returns true if further calls to `wasmer_import_object_iter_next` will + * not return any new data + */ +bool wasmer_import_object_iter_at_end(wasmer_import_object_iter_t *import_object_iter); + +/** + * Frees the memory allocated by `wasmer_import_object_iterate_functions` + */ +void wasmer_import_object_iter_destroy(wasmer_import_object_iter_t *import_object_iter); + +/** + * Writes the next value to `import`. `WASMER_ERROR` is returned if there + * was an error or there's nothing left to return. + * + * To free the memory allocated here, pass the import to `wasmer_import_object_imports_destroy`. + * To check if the iterator is done, use `wasmer_import_object_iter_at_end`. + */ +wasmer_result_t wasmer_import_object_iter_next(wasmer_import_object_iter_t *import_object_iter, + wasmer_import_t *import); + +/** + * Create an iterator over the functions in the import object. + * Get the next import with `wasmer_import_object_iter_next` + * Free the iterator with `wasmer_import_object_iter_destroy` + */ +wasmer_import_object_iter_t *wasmer_import_object_iterate_functions(const wasmer_import_object_t *import_object); + +/** + * Creates a new empty import object. + * See also `wasmer_import_object_append` + */ +wasmer_import_object_t *wasmer_import_object_new(void); + +wasmer_result_t wasmer_instance_cache(wasmer_instance_t *instance, + const uint8_t **cache_bytes, + uint32_t *cache_len); + +/** + * Calls an exported function of a WebAssembly instance by `name` + * with the provided parameters. The exported function results are + * stored on the provided `results` pointer. + * + * This function returns `wasmer_result_t::WASMER_OK` upon success, + * `wasmer_result_t::WASMER_ERROR` otherwise. You can use + * `wasmer_last_error_message()` to get the generated error message. + * + * Potential errors are the following: + * + * * `instance` is a null pointer, + * * `name` is a null pointer, + * * `params` is a null pointer. + * + * Example of calling an exported function that needs two parameters, and returns one value: + * + * ```c + * // First argument. + * wasmer_value_t argument_one = { + * .tag = WASM_I32, + * .value.I32 = 3, + * }; + * + * // Second argument. + * wasmer_value_t argument_two = { + * .tag = WASM_I32, + * .value.I32 = 4, + * }; + * + * // First result. + * wasmer_value_t result_one; + * + * // All arguments and results. + * wasmer_value_t arguments[] = {argument_one, argument_two}; + * wasmer_value_t results[] = {result_one}; + * + * wasmer_result_t call_result = wasmer_instance_call( + * instance, // instance pointer + * "sum", // the exported function name + * arguments, // the arguments + * 2, // the number of arguments + * results, // the results + * 1 // the number of results + * ); + * + * if (call_result == WASMER_OK) { + * printf("Result is: %d\n", results[0].value.I32); + * } + * ``` + */ +wasmer_result_t wasmer_instance_call(wasmer_instance_t *instance, + const char *name, + const wasmer_value_t *params, + uint32_t params_len, + wasmer_value_t *results, + uint32_t results_len); + +/** + * Gets the data that can be hold by an instance. + * + * This function is complementary of + * `wasmer_instance_context_data_set()`. Please read its + * documentation. You can also read the documentation of + * `wasmer_instance_context_t` to get other examples. + * + * This function returns nothing if `ctx` is a null pointer. + */ +void *wasmer_instance_context_data_get(const wasmer_instance_context_t *ctx); + +/** + * Sets the data that can be hold by an instance context. + * + * An instance context (represented by the opaque + * `wasmer_instance_context_t` structure) can hold user-defined + * data. This function sets the data. This function is complementary + * of `wasmer_instance_context_data_get()`. + * + * This function does nothing if `instance` is a null pointer. + * + * Example: + * + * ```c + * // Define your own data. + * typedef struct { + * // … + * } my_data; + * + * // Allocate them and set them on the given instance. + * my_data *data = malloc(sizeof(my_data)); + * data->… = …; + * wasmer_instance_context_data_set(instance, (void*) my_data); + * + * // You can read your data. + * { + * my_data *data = (my_data*) wasmer_instance_context_data_get(wasmer_instance_context_get(instance)); + * // … + * } + * ``` + */ +void wasmer_instance_context_data_set(wasmer_instance_t *instance, + void *data_ptr); + +/** + * Returns the instance context. Learn more by looking at the + * `wasmer_instance_context_t` struct. + * + * This function returns `null` if `instance` is a null pointer. + * + * Example: + * + * ```c + * const wasmer_instance_context_get *context = wasmer_instance_context_get(instance); + * my_data *data = (my_data *) wasmer_instance_context_data_get(context); + * // Do something with `my_data`. + * ``` + * + * It is often useful with `wasmer_instance_context_data_set()`. + */ +const wasmer_instance_context_t *wasmer_instance_context_get(wasmer_instance_t *instance); + +/** + * Gets the `memory_idx`th memory of the instance. + * + * Note that the index is always `0` until multiple memories are supported. + * + * This function is mostly used inside host functions (aka imported + * functions) to read the instance memory. + * + * Example of a _host function_ that reads and prints a string based on a pointer and a length: + * + * ```c + * void print_string(const wasmer_instance_context_t *context, int32_t pointer, int32_t length) { + * // Get the 0th memory. + * const wasmer_memory_t *memory = wasmer_instance_context_memory(context, 0); + * + * // Get the memory data as a pointer. + * uint8_t *memory_bytes = wasmer_memory_data(memory); + * + * // Print what we assumed to be a string! + * printf("%.*s", length, memory_bytes + pointer); + * } + * ``` + */ +const wasmer_memory_t *wasmer_instance_context_memory(const wasmer_instance_context_t *ctx, + uint32_t _memory_idx); + +/** + * Frees memory for the given `wasmer_instance_t`. + * + * Check the `wasmer_instantiate()` function to get a complete + * example. + * + * If `instance` is a null pointer, this function does nothing. + * + * Example: + * + * ```c + * // Get an instance. + * wasmer_instance_t *instance = NULL; + * wasmer_instantiate(&instance, bytes, bytes_length, imports, 0); + * + * // Destroy the instance. + * wasmer_instance_destroy(instance); + * ``` + */ +void wasmer_instance_destroy(wasmer_instance_t *instance); + +void wasmer_instance_disable_rkyv(void); + +void wasmer_instance_enable_rkyv(void); + +/** + * Gets all the exports of the given WebAssembly instance. + * + * This function stores a Rust vector of exports into `exports` as an + * opaque pointer of kind `wasmer_exports_t`. + * + * As is, you can do anything with `exports` except using the + * companion functions, like `wasmer_exports_len()`, + * `wasmer_exports_get()` or `wasmer_export_kind()`. See the example below. + * + * **Warning**: The caller owns the object and should call + * `wasmer_exports_destroy()` to free it. + * + * Example: + * + * ```c + * // Get the exports. + * wasmer_exports_t *exports = NULL; + * wasmer_instance_exports(instance, &exports); + * + * // Get the number of exports. + * int exports_length = wasmer_exports_len(exports); + * printf("Number of exports: %d\n", exports_length); + * + * // Read the first export. + * wasmer_export_t *export = wasmer_exports_get(exports, 0); + * + * // Get the kind of the export. + * wasmer_import_export_kind export_kind = wasmer_export_kind(export); + * + * // Assert it is a function (why not). + * assert(export_kind == WASM_FUNCTION); + * + * // Read the export name. + * wasmer_byte_array name_bytes = wasmer_export_name(export); + * + * assert(name_bytes.bytes_len == sizeof("sum") - 1); + * assert(memcmp(name_bytes.bytes, "sum", sizeof("sum") - 1) == 0); + * + * // Destroy the exports. + * wasmer_exports_destroy(exports); + * ``` + */ +void wasmer_instance_exports(wasmer_instance_t *instance, wasmer_exports_t **exports); + +wasmer_result_t wasmer_instance_from_cache(wasmer_instance_t **instance, + uint8_t *cache_bytes, + uint32_t cache_len, + const wasmer_compilation_options_t *options); + +uint64_t wasmer_instance_get_points_used(wasmer_instance_t *instance); + +uint64_t wasmer_instance_get_runtime_breakpoint_value(wasmer_instance_t *instance); + +/** + * Verifies whether the specified function name is imported by the given instance. + */ +bool wasmer_instance_is_function_imported(wasmer_instance_t *instance, const char *name); + +void wasmer_instance_set_points_limit(wasmer_instance_t *instance, uint64_t limit); + +void wasmer_instance_set_points_used(wasmer_instance_t *instance, uint64_t new_gas); + +void wasmer_instance_set_runtime_breakpoint_value(wasmer_instance_t *instance, uint64_t value); + +/** + * Creates a new WebAssembly instance from the given bytes and imports. + * + * The result is stored in the first argument `instance` if + * successful, i.e. when the function returns + * `wasmer_result_t::WASMER_OK`. Otherwise + * `wasmer_result_t::WASMER_ERROR` is returned, and + * `wasmer_last_error_length()` with `wasmer_last_error_message()` must + * be used to read the error message. + * + * The caller is responsible to free the instance with + * `wasmer_instance_destroy()`. + * + * Example: + * + * ```c + * // 1. Read a WebAssembly module from a file. + * FILE *file = fopen("sum.wasm", "r"); + * fseek(file, 0, SEEK_END); + * long bytes_length = ftell(file); + * uint8_t *bytes = malloc(bytes_length); + * fseek(file, 0, SEEK_SET); + * fread(bytes, 1, bytes_length, file); + * fclose(file); + * + * // 2. Declare the imports (here, none). + * wasmer_import_t imports[] = {}; + * + * // 3. Instantiate the WebAssembly module. + * wasmer_instance_t *instance = NULL; + * wasmer_result_t result = wasmer_instantiate(&instance, bytes, bytes_length, imports, 0); + * + * // 4. Check for errors. + * if (result != WASMER_OK) { + * int error_length = wasmer_last_error_length(); + * char *error = malloc(error_length); + * wasmer_last_error_message(error, error_length); + * // Do something with `error`… + * } + * + * // 5. Free the memory! + * wasmer_instance_destroy(instance); + * ``` + */ +wasmer_result_t wasmer_instantiate(wasmer_instance_t **instance, + uint8_t *wasm_bytes, + uint32_t wasm_bytes_len, + wasmer_import_t *imports, + int imports_len); + +wasmer_result_t wasmer_instantiate_with_options(wasmer_instance_t **instance, + uint8_t *wasm_bytes, + uint32_t wasm_bytes_len, + const wasmer_compilation_options_t *options); + +/** + * Gets the length in bytes of the last error if any. + * + * This can be used to dynamically allocate a buffer with the correct number of + * bytes needed to store a message. + * + * See `wasmer_last_error_message()` to get a full example. + */ +int wasmer_last_error_length(void); + +/** + * Gets the last error message if any into the provided buffer + * `buffer` up to the given `length`. + * + * The `length` parameter must be large enough to store the last + * error message. Ideally, the value should come from + * `wasmer_last_error_length()`. + * + * The function returns the length of the string in bytes, `-1` if an + * error occurs. Potential errors are: + * + * * The buffer is a null pointer, + * * The buffer is too smal to hold the error message. + * + * Note: The error message always has a trailing null character. + * + * Example: + * + * ```c + * int error_length = wasmer_last_error_length(); + * + * if (error_length > 0) { + * char *error_message = malloc(error_length); + * wasmer_last_error_message(error_message, error_length); + * printf("Error message: `%s`\n", error_message); + * } else { + * printf("No error message\n"); + * } + * ``` + */ +int wasmer_last_error_message(char *buffer, int length); + +/** + * Gets a pointer to the beginning of the contiguous memory data + * bytes. + * + * The function returns `NULL` if `memory` is a null pointer. + * + * Note that when the memory grows, it can be reallocated, and thus + * the returned pointer can be invalidated. + * + * Example: + * + * ```c + * uint8_t *memory_data = wasmer_memory_data(memory); + * char *str = (char*) malloc(sizeof(char) * 7); + * + * for (uint32_t nth = 0; nth < 7; ++nth) { + * str[nth] = (char) memory_data[nth]; + * } + * ``` + */ +uint8_t *wasmer_memory_data(const wasmer_memory_t *memory); + +/** + * Gets the size in bytes of the memory data. + * + * This function returns 0 if `memory` is a null pointer. + * + * Example: + * + * ```c + * uint32_t memory_data_length = wasmer_memory_data_length(memory); + * ``` + */ +uint32_t wasmer_memory_data_length(wasmer_memory_t *memory); + +/** + * Frees memory for the given `wasmer_memory_t`. + * + * Check the `wasmer_memory_new()` function to get a complete + * example. + * + * If `memory` is a null pointer, this function does nothing. + * + * Example: + * + * ```c + * // Get a memory. + * wasmer_memory_t *memory = NULL; + * wasmer_result_t result = wasmer_memory_new(&memory, memory_descriptor); + * + * // Destroy the memory. + * wasmer_memory_destroy(memory); + * ``` + */ +void wasmer_memory_destroy(wasmer_memory_t *memory); + +/** + * Grows a memory by the given number of pages (of 65Kb each). + * + * The functions return `wasmer_result_t::WASMER_OK` upon success, + * `wasmer_result_t::WASMER_ERROR` otherwise. Use + * `wasmer_last_error_length()` with `wasmer_last_error_message()` to + * read the error message. + * + * Example: + * + * ```c + * wasmer_result_t result = wasmer_memory_grow(memory, 10); + * + * if (result != WASMER_OK) { + * // … + * } + * ``` + */ +wasmer_result_t wasmer_memory_grow(wasmer_memory_t *memory, uint32_t delta); + +/** + * Reads the current length (in pages) of the given memory. + * + * The function returns zero if `memory` is a null pointer. + * + * Example: + * + * ```c + * uint32_t memory_length = wasmer_memory_length(memory); + * + * printf("Memory pages length: %d\n", memory_length); + * ``` + */ +uint32_t wasmer_memory_length(const wasmer_memory_t *memory); + +/** + * Creates a new empty WebAssembly memory for the given descriptor. + * + * The result is stored in the first argument `memory` if successful, + * i.e. when the function returns + * `wasmer_result_t::WASMER_OK`. Otherwise, + * `wasmer_result_t::WASMER_ERROR` is returned, and + * `wasmer_last_error_length()` with `wasmer_last_error_message()` + * must be used to read the error message. + * + * The caller owns the memory and is responsible to free it with + * `wasmer_memory_destroy()`. + * + * Example: + * + * ```c + * // 1. The memory object. + * wasmer_memory_t *memory = NULL; + * + * // 2. The memory descriptor. + * wasmer_limits_t memory_descriptor = { + * .min = 10, + * .max = { + * .has_some = true, + * .some = 15, + * }, + * }; + * + * // 3. Initialize the memory. + * wasmer_result_t result = wasmer_memory_new(&memory, memory_descriptor); + * + * if (result != WASMER_OK) { + * int error_length = wasmer_last_error_length(); + * char *error = malloc(error_length); + * wasmer_last_error_message(error, error_length); + * // Do something with `error`… + * } + * + * // 4. Free the memory! + * wasmer_memory_destroy(memory); + * ``` + */ +wasmer_result_t wasmer_memory_new(wasmer_memory_t **memory, wasmer_limits_t limits); + +/** + * Deserialize the given serialized module. + * + * Returns `wasmer_result_t::WASMER_OK` upon success. + * + * Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` + * and `wasmer_last_error_message` to get an error message. + */ +wasmer_result_t wasmer_module_deserialize(wasmer_module_t **module, + const wasmer_serialized_module_t *serialized_module); + +/** + * Frees memory for the given Module + */ +void wasmer_module_destroy(wasmer_module_t *module); + +/** + * Given: + * * A prepared `wasmer` import-object + * * A compiled wasmer module + * + * Instantiates a wasmer instance + */ +wasmer_result_t wasmer_module_import_instantiate(wasmer_instance_t **instance, + const wasmer_module_t *module, + const wasmer_import_object_t *import_object); + +/** + * Creates a new Instance from the given module and imports. + * + * Returns `wasmer_result_t::WASMER_OK` upon success. + * + * Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` + * and `wasmer_last_error_message` to get an error message. + */ +wasmer_result_t wasmer_module_instantiate(const wasmer_module_t *module, + wasmer_instance_t **instance, + wasmer_import_t *imports, + int imports_len); + +/** + * Serialize the given Module. + * + * The caller owns the object and should call `wasmer_serialized_module_destroy` to free it. + * + * Returns `wasmer_result_t::WASMER_OK` upon success. + * + * Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` + * and `wasmer_last_error_message` to get an error message. + */ +wasmer_result_t wasmer_module_serialize(wasmer_serialized_module_t **serialized_module, + const wasmer_module_t *module); + +/** + * Get bytes of the serialized module. + */ +wasmer_byte_array wasmer_serialized_module_bytes(const wasmer_serialized_module_t *serialized_module); + +/** + * Frees memory for the given serialized Module. + */ +void wasmer_serialized_module_destroy(wasmer_serialized_module_t *serialized_module); + +/** + * Transform a sequence of bytes into a serialized module. + * + * The caller owns the object and should call `wasmer_serialized_module_destroy` to free it. + * + * Returns `wasmer_result_t::WASMER_OK` upon success. + * + * Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` + * and `wasmer_last_error_message` to get an error message. + */ +wasmer_result_t wasmer_serialized_module_from_bytes(wasmer_serialized_module_t **serialized_module, + const uint8_t *serialized_module_bytes, + uint32_t serialized_module_bytes_length); + +void wasmer_set_opcode_costs(const uint32_t *opcode_costs_pointer); + +void wasmer_set_sigsegv_passthrough(void); + +/** + * Frees memory for the given Table + */ +void wasmer_table_destroy(wasmer_table_t *table); + +/** + * Grows a Table by the given number of elements. + * + * Returns `wasmer_result_t::WASMER_OK` upon success. + * + * Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` + * and `wasmer_last_error_message` to get an error message. + */ +wasmer_result_t wasmer_table_grow(wasmer_table_t *table, uint32_t delta); + +/** + * Returns the current length of the given Table + */ +uint32_t wasmer_table_length(wasmer_table_t *table); + +/** + * Creates a new Table for the given descriptor and initializes the given + * pointer to pointer to a pointer to the new Table. + * + * The caller owns the object and should call `wasmer_table_destroy` to free it. + * + * Returns `wasmer_result_t::WASMER_OK` upon success. + * + * Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` + * and `wasmer_last_error_message` to get an error message. + */ +wasmer_result_t wasmer_table_new(wasmer_table_t **table, wasmer_limits_t limits); + +#if (!defined(_WIN32) && defined(ARCH_X86_64)) +/** + * Adds a callinfo trampoline to the builder. + */ +uintptr_t wasmer_trampoline_buffer_builder_add_callinfo_trampoline(wasmer_trampoline_buffer_builder_t *builder, + const wasmer_trampoline_callable_t *func, + const void *ctx, + uint32_t num_params); +#endif + +#if (!defined(_WIN32) && defined(ARCH_X86_64)) +/** + * Adds a context trampoline to the builder. + */ +uintptr_t wasmer_trampoline_buffer_builder_add_context_trampoline(wasmer_trampoline_buffer_builder_t *builder, + const wasmer_trampoline_callable_t *func, + const void *ctx); +#endif + +#if (!defined(_WIN32) && defined(ARCH_X86_64)) +/** + * Finalizes the trampoline builder into an executable buffer. + */ +wasmer_trampoline_buffer_t *wasmer_trampoline_buffer_builder_build(wasmer_trampoline_buffer_builder_t *builder); +#endif + +#if (!defined(_WIN32) && defined(ARCH_X86_64)) +/** + * Creates a new trampoline builder. + */ +wasmer_trampoline_buffer_builder_t *wasmer_trampoline_buffer_builder_new(void); +#endif + +#if (!defined(_WIN32) && defined(ARCH_X86_64)) +/** + * Destroys the trampoline buffer if not null. + */ +void wasmer_trampoline_buffer_destroy(wasmer_trampoline_buffer_t *buffer); +#endif + +#if (!defined(_WIN32) && defined(ARCH_X86_64)) +/** + * Returns the callable pointer for the trampoline with index `idx`. + */ +const wasmer_trampoline_callable_t *wasmer_trampoline_buffer_get_trampoline(const wasmer_trampoline_buffer_t *buffer, + uintptr_t idx); +#endif + +#if (!defined(_WIN32) && defined(ARCH_X86_64)) +/** + * Returns the context added by `add_context_trampoline`, from within the callee function. + */ +void *wasmer_trampoline_get_context(void); +#endif + +/** + * Stop the execution of a host function, aka imported function. The + * function must be used _only_ inside a host function. + * + * The pointer to `wasmer_instance_context_t` is received by the host + * function as its first argument. Just passing it to `ctx` is fine. + * + * The error message must have a greater lifetime than the host + * function itself since the error is read outside the host function + * with `wasmer_last_error_message`. + * + * This function returns `wasmer_result_t::WASMER_ERROR` if `ctx` or + * `error_message` are null. + * + * This function never returns otherwise. + */ +wasmer_result_t wasmer_trap(const wasmer_instance_context_t *ctx, const char *error_message); + +/** + * Validates a sequence of bytes hoping it represents a valid WebAssembly module. + * + * The function returns true if the bytes are valid, false otherwise. + * + * Example: + * + * ```c + * bool result = wasmer_validate(bytes, bytes_length); + * + * if (false == result) { + * // Do something… + * } + * ``` + */ +bool wasmer_validate(const uint8_t *wasm_bytes, uint32_t wasm_bytes_len); + +#if defined(WASMER_WASI_ENABLED) +/** + * Convenience function that creates a WASI import object with no arguments, + * environment variables, preopened files, or mapped directories. + * + * This function is the same as calling [`wasmer_wasi_generate_import_object`] with all + * empty values. + */ +wasmer_import_object_t *wasmer_wasi_generate_default_import_object(void); +#endif + +#if defined(WASMER_WASI_ENABLED) +/** + * Creates a WASI import object. + * + * This function treats null pointers as empty collections. + * For example, passing null for a string in `args`, will lead to a zero + * length argument in that position. + */ +wasmer_import_object_t *wasmer_wasi_generate_import_object(const wasmer_byte_array *args, + unsigned int args_len, + const wasmer_byte_array *envs, + unsigned int envs_len, + const wasmer_byte_array *preopened_files, + unsigned int preopened_files_len, + const wasmer_wasi_map_dir_entry_t *mapped_dirs, + unsigned int mapped_dirs_len); +#endif + +#if defined(WASMER_WASI_ENABLED) +/** + * Creates a WASI import object for a specific version. + * + * This function is similar to `wasmer_wasi_generate_import_object` + * except that the first argument describes the WASI version. + * + * The version is expected to be of kind `Version`. + */ +wasmer_import_object_t *wasmer_wasi_generate_import_object_for_version(unsigned char version, + const wasmer_byte_array *args, + unsigned int args_len, + const wasmer_byte_array *envs, + unsigned int envs_len, + const wasmer_byte_array *preopened_files, + unsigned int preopened_files_len, + const wasmer_wasi_map_dir_entry_t *mapped_dirs, + unsigned int mapped_dirs_len); +#endif + +#if defined(WASMER_WASI_ENABLED) +/** + * Find the version of WASI used by the module. + * + * In case of error, the returned version is `Version::Unknown`. + */ +Version wasmer_wasi_get_version(const wasmer_module_t *module); +#endif + +#endif /* WASMER_H */ diff --git a/lib/runtime-c-api/wasmer.hh b/lib/runtime-c-api/wasmer.hh new file mode 100644 index 000000000000..22725ec7dd7d --- /dev/null +++ b/lib/runtime-c-api/wasmer.hh @@ -0,0 +1,1328 @@ + +#if !defined(WASMER_H_MACROS) + +#define WASMER_H_MACROS + +// Define the `ARCH_X86_X64` constant. +#if defined(MSVC) && defined(_M_AMD64) +# define ARCH_X86_64 +#elif (defined(GCC) || defined(__GNUC__) || defined(__clang__)) && defined(__x86_64__) +# define ARCH_X86_64 +#endif + +// Compatibility with non-Clang compilers. +#if !defined(__has_attribute) +# define __has_attribute(x) 0 +#endif + +// Compatibility with non-Clang compilers. +#if !defined(__has_declspec_attribute) +# define __has_declspec_attribute(x) 0 +#endif + +// Define the `DEPRECATED` macro. +#if defined(GCC) || defined(__GNUC__) || __has_attribute(deprecated) +# define DEPRECATED(message) __attribute__((deprecated(message))) +#elif defined(MSVC) || __has_declspec_attribute(deprecated) +# define DEPRECATED(message) __declspec(deprecated(message)) +#endif + +#endif // WASMER_H_MACROS + + +#ifndef WASMER_H +#define WASMER_H + +#include +#include +#include +#include + +static const uintptr_t OPCODE_COUNT = 448; + +#if defined(WASMER_WASI_ENABLED) +enum class Version : uint8_t { + /// Version cannot be detected or is unknown. + Unknown = 0, + /// Latest version. See `wasmer_wasi::WasiVersion::Latest` to + /// learn more. + Latest = 1, + /// `wasi_unstable`. + Snapshot0 = 2, + /// `wasi_snapshot_preview1`. + Snapshot1 = 3, +}; +#endif + +/// List of export/import kinds. +enum class wasmer_import_export_kind : uint32_t { + /// The export/import is a function. + WASM_FUNCTION = 0, + /// The export/import is a global. + WASM_GLOBAL = 1, + /// The export/import is a memory. + WASM_MEMORY = 2, + /// The export/import is a table. + WASM_TABLE = 3, +}; + +/// The `wasmer_result_t` enum is a type that represents either a +/// success, or a failure. +enum class wasmer_result_t { + /// Represents a success. + WASMER_OK = 1, + /// Represents a failure. + WASMER_ERROR = 2, +}; + +/// Represents all possibles WebAssembly value types. +/// +/// See `wasmer_value_t` to get a complete example. +enum class wasmer_value_tag : uint32_t { + /// Represents the `i32` WebAssembly type. + WASM_I32, + /// Represents the `i64` WebAssembly type. + WASM_I64, + /// Represents the `f32` WebAssembly type. + WASM_F32, + /// Represents the `f64` WebAssembly type. + WASM_F64, +}; + +struct wasmer_module_t { + +}; + +/// Opaque pointer to a `wasmer_runtime::Instance` value in Rust. +/// +/// A `wasmer_runtime::Instance` represents a WebAssembly instance. It +/// is generally generated by the `wasmer_instantiate()` function, or by +/// the `wasmer_module_instantiate()` function for the most common paths. +struct wasmer_instance_t { + +}; + +struct wasmer_byte_array { + const uint8_t *bytes; + uint32_t bytes_len; +}; + +#if defined(WASMER_EMSCRIPTEN_ENABLED) +/// Type used to construct an import_object_t with Emscripten imports. +struct wasmer_emscripten_globals_t { + +}; +#endif + +struct wasmer_import_object_t { + +}; + +/// Opaque pointer to `NamedExportDescriptor`. +struct wasmer_export_descriptor_t { + +}; + +/// Opaque pointer to `NamedExportDescriptors`. +struct wasmer_export_descriptors_t { + +}; + +/// Opaque pointer to `wasmer_export_t`. +struct wasmer_export_func_t { + +}; + +/// Represents a WebAssembly value. +/// +/// This is a [Rust union][rust-union], which is equivalent to the C +/// union. See `wasmer_value_t` to get a complete example. +/// +/// [rust-union]: https://doc.rust-lang.org/reference/items/unions.html +union wasmer_value { + int32_t I32; + int64_t I64; + float F32; + double F64; +}; + +/// Represents a WebAssembly type and value pair, +/// i.e. `wasmer_value_tag` and `wasmer_value`. Since the latter is an +/// union, it's the safe way to read or write a WebAssembly value in +/// C. +/// +/// Example: +/// +/// ```c +/// // Create a WebAssembly value. +/// wasmer_value_t wasm_value = { +/// .tag = WASM_I32, +/// .value.I32 = 42, +/// }; +/// +/// // Read a WebAssembly value. +/// if (wasm_value.tag == WASM_I32) { +/// int32_t x = wasm_value.value.I32; +/// // … +/// } +/// ``` +struct wasmer_value_t { + /// The value type. + wasmer_value_tag tag; + /// The value. + wasmer_value value; +}; + +/// Opaque pointer to `NamedExport`. +struct wasmer_export_t { + +}; + +/// Opaque pointer to a `wasmer_runtime::Memory` value in Rust. +/// +/// A `wasmer_runtime::Memory` represents a WebAssembly memory. It is +/// possible to create one with `wasmer_memory_new()` and pass it as +/// imports of an instance, or to read it from exports of an instance +/// with `wasmer_export_to_memory()`. +struct wasmer_memory_t { + +}; + +/// Opaque pointer to the opaque structure `crate::NamedExports`, +/// which is a wrapper around a vector of the opaque structure +/// `crate::NamedExport`. +/// +/// Check the `wasmer_instance_exports()` function to learn more. +struct wasmer_exports_t { + +}; + +struct wasmer_global_t { + +}; + +struct wasmer_global_descriptor_t { + bool mutable_; + wasmer_value_tag kind; +}; + +struct wasmer_import_descriptor_t { + +}; + +struct wasmer_import_descriptors_t { + +}; + +struct wasmer_import_func_t { + +}; + +struct wasmer_table_t { + +}; + +/// Union of import/export value. +union wasmer_import_export_value { + const wasmer_import_func_t *func; + const wasmer_table_t *table; + const wasmer_memory_t *memory; + const wasmer_global_t *global; +}; + +struct wasmer_import_t { + wasmer_byte_array module_name; + wasmer_byte_array import_name; + wasmer_import_export_kind tag; + wasmer_import_export_value value; +}; + +struct wasmer_import_object_iter_t { + +}; + +/// Opaque pointer to a `wasmer_runtime::Ctx` value in Rust. +/// +/// An instance context is passed to any host function (aka imported +/// function) as the first argument. It is necessary to read the +/// instance data or the memory, respectively with the +/// `wasmer_instance_context_data_get()` function, and the +/// `wasmer_instance_context_memory()` function. +/// +/// It is also possible to get the instance context outside a host +/// function by using the `wasmer_instance_context_get()` +/// function. See also `wasmer_instance_context_data_set()` to set the +/// instance context data. +/// +/// Example: +/// +/// ```c +/// // A host function that prints data from the WebAssembly memory to +/// // the standard output. +/// void print(wasmer_instance_context_t *context, int32_t pointer, int32_t length) { +/// // Use `wasmer_instance_context` to get back the first instance memory. +/// const wasmer_memory_t *memory = wasmer_instance_context_memory(context, 0); +/// +/// // Continue… +/// } +/// ``` +struct wasmer_instance_context_t { + +}; + +struct wasmer_compilation_options_t { + +}; + +/// The `wasmer_limit_option_t` struct represents an optional limit +/// for `wasmer_limits_t`. +struct wasmer_limit_option_t { + /// Whether the limit is set. + bool has_some; + /// The limit value. + uint32_t some; +}; + +/// The `wasmer_limits_t` struct is a type that describes a memory +/// options. See the `wasmer_memory_t` struct or the +/// `wasmer_memory_new()` function to get more information. +struct wasmer_limits_t { + /// The minimum number of allowed pages. + uint32_t min; + /// The maximum number of allowed pages. + wasmer_limit_option_t max; +}; + +struct wasmer_serialized_module_t { + +}; + +#if (!defined(_WIN32) && defined(ARCH_X86_64)) +struct wasmer_trampoline_buffer_builder_t { + +}; +#endif + +#if (!defined(_WIN32) && defined(ARCH_X86_64)) +struct wasmer_trampoline_callable_t { + +}; +#endif + +#if (!defined(_WIN32) && defined(ARCH_X86_64)) +struct wasmer_trampoline_buffer_t { + +}; +#endif + +#if defined(WASMER_WASI_ENABLED) +/// Opens a directory that's visible to the WASI module as `alias` but +/// is backed by the host file at `host_file_path` +struct wasmer_wasi_map_dir_entry_t { + /// What the WASI module will see in its virtual root + wasmer_byte_array alias; + /// The backing file that the WASI module will interact with via the alias + wasmer_byte_array host_file_path; +}; +#endif + +extern "C" { + +/// Creates a new Module from the given wasm bytes. +/// +/// Returns `wasmer_result_t::WASMER_OK` upon success. +/// +/// Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` +/// and `wasmer_last_error_message` to get an error message. +wasmer_result_t wasmer_compile(wasmer_module_t **module, + uint8_t *wasm_bytes, + uint32_t wasm_bytes_len); + +/// Creates a new Module with gas limit from the given wasm bytes. +/// +/// Returns `wasmer_result_t::WASMER_OK` upon success. +/// +/// Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` +/// and `wasmer_last_error_message` to get an error message. +wasmer_result_t wasmer_compile_with_gas_metering(wasmer_module_t **module, + uint8_t *wasm_bytes, + uint32_t wasm_bytes_len); + +wasmer_result_t wasmer_compile_with_gas_metering(wasmer_module_t **module, + uint8_t *wasm_bytes, + uint32_t wasm_bytes_len); + +#if defined(WASMER_EMSCRIPTEN_ENABLED) +/// Convenience function for setting up arguments and calling the Emscripten +/// main function. +/// +/// WARNING: +/// +/// Do not call this function on untrusted code when operating without +/// additional sandboxing in place. +/// Emscripten has access to many host system calls and therefore may do very +/// bad things. +wasmer_result_t wasmer_emscripten_call_main(wasmer_instance_t *instance, + const wasmer_byte_array *args, + unsigned int args_len); +#endif + +#if defined(WASMER_EMSCRIPTEN_ENABLED) +/// Destroy `wasmer_emscrpten_globals_t` created by +/// `wasmer_emscripten_get_emscripten_globals`. +void wasmer_emscripten_destroy_globals(wasmer_emscripten_globals_t *globals); +#endif + +#if defined(WASMER_EMSCRIPTEN_ENABLED) +/// Create a `wasmer_import_object_t` with Emscripten imports, use +/// `wasmer_emscripten_get_emscripten_globals` to get a +/// `wasmer_emscripten_globals_t` from a `wasmer_module_t`. +/// +/// WARNING: +/// +/// This `import_object_t` contains thin-wrappers around host system calls. +/// Do not use this to execute untrusted code without additional sandboxing. +wasmer_import_object_t *wasmer_emscripten_generate_import_object(wasmer_emscripten_globals_t *globals); +#endif + +#if defined(WASMER_EMSCRIPTEN_ENABLED) +/// Create a `wasmer_emscripten_globals_t` from a Wasm module. +wasmer_emscripten_globals_t *wasmer_emscripten_get_globals(const wasmer_module_t *module); +#endif + +#if defined(WASMER_EMSCRIPTEN_ENABLED) +/// Execute global constructors (required if the module is compiled from C++) +/// and sets up the internal environment. +/// +/// This function sets the data pointer in the same way that +/// [`wasmer_instance_context_data_set`] does. +wasmer_result_t wasmer_emscripten_set_up(wasmer_instance_t *instance, + wasmer_emscripten_globals_t *globals); +#endif + +/// Gets export descriptor kind +wasmer_import_export_kind wasmer_export_descriptor_kind(wasmer_export_descriptor_t *export_); + +/// Gets name for the export descriptor +wasmer_byte_array wasmer_export_descriptor_name(wasmer_export_descriptor_t *export_descriptor); + +/// Gets export descriptors for the given module +/// +/// The caller owns the object and should call `wasmer_export_descriptors_destroy` to free it. +void wasmer_export_descriptors(const wasmer_module_t *module, + wasmer_export_descriptors_t **export_descriptors); + +/// Frees the memory for the given export descriptors +void wasmer_export_descriptors_destroy(wasmer_export_descriptors_t *export_descriptors); + +/// Gets export descriptor by index +wasmer_export_descriptor_t *wasmer_export_descriptors_get(wasmer_export_descriptors_t *export_descriptors, + int idx); + +/// Gets the length of the export descriptors +int wasmer_export_descriptors_len(wasmer_export_descriptors_t *exports); + +/// Calls a `func` with the provided parameters. +/// Results are set using the provided `results` pointer. +/// +/// Returns `wasmer_result_t::WASMER_OK` upon success. +/// +/// Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` +/// and `wasmer_last_error_message` to get an error message. +wasmer_result_t wasmer_export_func_call(const wasmer_export_func_t *func, + const wasmer_value_t *params, + unsigned int params_len, + wasmer_value_t *results, + unsigned int results_len); + +/// Sets the params buffer to the parameter types of the given wasmer_export_func_t +/// +/// Returns `wasmer_result_t::WASMER_OK` upon success. +/// +/// Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` +/// and `wasmer_last_error_message` to get an error message. +wasmer_result_t wasmer_export_func_params(const wasmer_export_func_t *func, + wasmer_value_tag *params, + uint32_t params_len); + +/// Sets the result parameter to the arity of the params of the wasmer_export_func_t +/// +/// Returns `wasmer_result_t::WASMER_OK` upon success. +/// +/// Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` +/// and `wasmer_last_error_message` to get an error message. +wasmer_result_t wasmer_export_func_params_arity(const wasmer_export_func_t *func, uint32_t *result); + +/// Sets the returns buffer to the parameter types of the given wasmer_export_func_t +/// +/// Returns `wasmer_result_t::WASMER_OK` upon success. +/// +/// Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` +/// and `wasmer_last_error_message` to get an error message. +wasmer_result_t wasmer_export_func_returns(const wasmer_export_func_t *func, + wasmer_value_tag *returns, + uint32_t returns_len); + +/// Sets the result parameter to the arity of the returns of the wasmer_export_func_t +/// +/// Returns `wasmer_result_t::WASMER_OK` upon success. +/// +/// Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` +/// and `wasmer_last_error_message` to get an error message. +wasmer_result_t wasmer_export_func_returns_arity(const wasmer_export_func_t *func, + uint32_t *result); + +/// Gets wasmer_export kind +wasmer_import_export_kind wasmer_export_kind(wasmer_export_t *export_); + +/// Gets name from wasmer_export +wasmer_byte_array wasmer_export_name(wasmer_export_t *export_); + +/// Gets export func from export +const wasmer_export_func_t *wasmer_export_to_func(const wasmer_export_t *export_); + +/// Gets a memory pointer from an export pointer. +/// +/// Returns `wasmer_result_t::WASMER_OK` upon success. +/// +/// Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` +/// and `wasmer_last_error_message` to get an error message. +wasmer_result_t wasmer_export_to_memory(const wasmer_export_t *export_, wasmer_memory_t **memory); + +/// Frees the memory for the given exports. +/// +/// Check the `wasmer_instance_exports()` function to get a complete +/// example. +/// +/// If `exports` is a null pointer, this function does nothing. +/// +/// Example: +/// +/// ```c +/// // Get some exports. +/// wasmer_exports_t *exports = NULL; +/// wasmer_instance_exports(instance, &exports); +/// +/// // Destroy the exports. +/// wasmer_exports_destroy(exports); +/// ``` +void wasmer_exports_destroy(wasmer_exports_t *exports); + +/// Gets wasmer_export by index +wasmer_export_t *wasmer_exports_get(wasmer_exports_t *exports, int idx); + +/// Gets the length of the exports +int wasmer_exports_len(wasmer_exports_t *exports); + +/// Frees memory for the given Global +void wasmer_global_destroy(wasmer_global_t *global); + +/// Gets the value stored by the given Global +wasmer_value_t wasmer_global_get(wasmer_global_t *global); + +/// Returns a descriptor (type, mutability) of the given Global +wasmer_global_descriptor_t wasmer_global_get_descriptor(wasmer_global_t *global); + +/// Creates a new Global and returns a pointer to it. +/// The caller owns the object and should call `wasmer_global_destroy` to free it. +wasmer_global_t *wasmer_global_new(wasmer_value_t value, bool mutable_); + +/// Sets the value stored by the given Global +void wasmer_global_set(wasmer_global_t *global, wasmer_value_t value); + +/// Gets export descriptor kind +wasmer_import_export_kind wasmer_import_descriptor_kind(wasmer_import_descriptor_t *export_); + +/// Gets module name for the import descriptor +wasmer_byte_array wasmer_import_descriptor_module_name(wasmer_import_descriptor_t *import_descriptor); + +/// Gets name for the import descriptor +wasmer_byte_array wasmer_import_descriptor_name(wasmer_import_descriptor_t *import_descriptor); + +/// Gets import descriptors for the given module +/// +/// The caller owns the object and should call `wasmer_import_descriptors_destroy` to free it. +void wasmer_import_descriptors(const wasmer_module_t *module, + wasmer_import_descriptors_t **import_descriptors); + +/// Frees the memory for the given import descriptors +void wasmer_import_descriptors_destroy(wasmer_import_descriptors_t *import_descriptors); + +/// Gets import descriptor by index +wasmer_import_descriptor_t *wasmer_import_descriptors_get(wasmer_import_descriptors_t *import_descriptors, + unsigned int idx); + +/// Gets the length of the import descriptors +unsigned int wasmer_import_descriptors_len(wasmer_import_descriptors_t *exports); + +/// Frees memory for the given Func +void wasmer_import_func_destroy(wasmer_import_func_t *func); + +/// Creates new host function, aka imported function. `func` is a +/// function pointer, where the first argument is the famous `vm::Ctx` +/// (in Rust), or `wasmer_instance_context_t` (in C). All arguments +/// must be typed with compatible WebAssembly native types: +/// +/// | WebAssembly type | C/C++ type | +/// | ---------------- | ---------- | +/// | `i32` | `int32_t` | +/// | `i64` | `int64_t` | +/// | `f32` | `float` | +/// | `f64` | `double` | +/// +/// The function pointer must have a lifetime greater than the +/// WebAssembly instance lifetime. +/// +/// The caller owns the object and should call +/// `wasmer_import_func_destroy` to free it. +wasmer_import_func_t *wasmer_import_func_new(void (*func)(void *data), + const wasmer_value_tag *params, + unsigned int params_len, + const wasmer_value_tag *returns, + unsigned int returns_len); + +/// Sets the params buffer to the parameter types of the given wasmer_import_func_t +/// +/// Returns `wasmer_result_t::WASMER_OK` upon success. +/// +/// Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` +/// and `wasmer_last_error_message` to get an error message. +wasmer_result_t wasmer_import_func_params(const wasmer_import_func_t *func, + wasmer_value_tag *params, + unsigned int params_len); + +/// Sets the result parameter to the arity of the params of the wasmer_import_func_t +/// +/// Returns `wasmer_result_t::WASMER_OK` upon success. +/// +/// Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` +/// and `wasmer_last_error_message` to get an error message. +wasmer_result_t wasmer_import_func_params_arity(const wasmer_import_func_t *func, uint32_t *result); + +/// Sets the returns buffer to the parameter types of the given wasmer_import_func_t +/// +/// Returns `wasmer_result_t::WASMER_OK` upon success. +/// +/// Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` +/// and `wasmer_last_error_message` to get an error message. +wasmer_result_t wasmer_import_func_returns(const wasmer_import_func_t *func, + wasmer_value_tag *returns, + unsigned int returns_len); + +/// Sets the result parameter to the arity of the returns of the wasmer_import_func_t +/// +/// Returns `wasmer_result_t::WASMER_OK` upon success. +/// +/// Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` +/// and `wasmer_last_error_message` to get an error message. +wasmer_result_t wasmer_import_func_returns_arity(const wasmer_import_func_t *func, + uint32_t *result); + +wasmer_result_t wasmer_import_object_cache_from_imports(wasmer_import_t *imports, + unsigned int imports_len); + +/// Frees memory of the given ImportObject +void wasmer_import_object_destroy(wasmer_import_object_t *import_object); + +/// Extends an existing import object with new imports +wasmer_result_t wasmer_import_object_extend(wasmer_import_object_t *import_object, + const wasmer_import_t *imports, + unsigned int imports_len); + +/// Gets an entry from an ImportObject at the name and namespace. +/// Stores `name`, `namespace`, and `import_export_value` in `import`. +/// Thus these must remain valid for the lifetime of `import`. +/// +/// The caller owns all data involved. +/// `import_export_value` will be written to based on `tag`. +wasmer_result_t wasmer_import_object_get_import(const wasmer_import_object_t *import_object, + wasmer_byte_array namespace_, + wasmer_byte_array name, + wasmer_import_t *import, + wasmer_import_export_value *import_export_value, + uint32_t tag); + +/// Frees the memory allocated in `wasmer_import_object_iter_next` +/// +/// This function does not free the memory in `wasmer_import_object_t`; +/// it only frees memory allocated while querying a `wasmer_import_object_t`. +void wasmer_import_object_imports_destroy(wasmer_import_t *imports, uint32_t imports_len); + +/// Returns true if further calls to `wasmer_import_object_iter_next` will +/// not return any new data +bool wasmer_import_object_iter_at_end(wasmer_import_object_iter_t *import_object_iter); + +/// Frees the memory allocated by `wasmer_import_object_iterate_functions` +void wasmer_import_object_iter_destroy(wasmer_import_object_iter_t *import_object_iter); + +/// Writes the next value to `import`. `WASMER_ERROR` is returned if there +/// was an error or there's nothing left to return. +/// +/// To free the memory allocated here, pass the import to `wasmer_import_object_imports_destroy`. +/// To check if the iterator is done, use `wasmer_import_object_iter_at_end`. +wasmer_result_t wasmer_import_object_iter_next(wasmer_import_object_iter_t *import_object_iter, + wasmer_import_t *import); + +/// Create an iterator over the functions in the import object. +/// Get the next import with `wasmer_import_object_iter_next` +/// Free the iterator with `wasmer_import_object_iter_destroy` +wasmer_import_object_iter_t *wasmer_import_object_iterate_functions(const wasmer_import_object_t *import_object); + +/// Creates a new empty import object. +/// See also `wasmer_import_object_append` +wasmer_import_object_t *wasmer_import_object_new(); + +wasmer_result_t wasmer_instance_cache(wasmer_instance_t *instance, + const uint8_t **cache_bytes, + uint32_t *cache_len); + +/// Calls an exported function of a WebAssembly instance by `name` +/// with the provided parameters. The exported function results are +/// stored on the provided `results` pointer. +/// +/// This function returns `wasmer_result_t::WASMER_OK` upon success, +/// `wasmer_result_t::WASMER_ERROR` otherwise. You can use +/// `wasmer_last_error_message()` to get the generated error message. +/// +/// Potential errors are the following: +/// +/// * `instance` is a null pointer, +/// * `name` is a null pointer, +/// * `params` is a null pointer. +/// +/// Example of calling an exported function that needs two parameters, and returns one value: +/// +/// ```c +/// // First argument. +/// wasmer_value_t argument_one = { +/// .tag = WASM_I32, +/// .value.I32 = 3, +/// }; +/// +/// // Second argument. +/// wasmer_value_t argument_two = { +/// .tag = WASM_I32, +/// .value.I32 = 4, +/// }; +/// +/// // First result. +/// wasmer_value_t result_one; +/// +/// // All arguments and results. +/// wasmer_value_t arguments[] = {argument_one, argument_two}; +/// wasmer_value_t results[] = {result_one}; +/// +/// wasmer_result_t call_result = wasmer_instance_call( +/// instance, // instance pointer +/// "sum", // the exported function name +/// arguments, // the arguments +/// 2, // the number of arguments +/// results, // the results +/// 1 // the number of results +/// ); +/// +/// if (call_result == WASMER_OK) { +/// printf("Result is: %d\n", results[0].value.I32); +/// } +/// ``` +wasmer_result_t wasmer_instance_call(wasmer_instance_t *instance, + const char *name, + const wasmer_value_t *params, + uint32_t params_len, + wasmer_value_t *results, + uint32_t results_len); + +/// Gets the data that can be hold by an instance. +/// +/// This function is complementary of +/// `wasmer_instance_context_data_set()`. Please read its +/// documentation. You can also read the documentation of +/// `wasmer_instance_context_t` to get other examples. +/// +/// This function returns nothing if `ctx` is a null pointer. +void *wasmer_instance_context_data_get(const wasmer_instance_context_t *ctx); + +/// Sets the data that can be hold by an instance context. +/// +/// An instance context (represented by the opaque +/// `wasmer_instance_context_t` structure) can hold user-defined +/// data. This function sets the data. This function is complementary +/// of `wasmer_instance_context_data_get()`. +/// +/// This function does nothing if `instance` is a null pointer. +/// +/// Example: +/// +/// ```c +/// // Define your own data. +/// typedef struct { +/// // … +/// } my_data; +/// +/// // Allocate them and set them on the given instance. +/// my_data *data = malloc(sizeof(my_data)); +/// data->… = …; +/// wasmer_instance_context_data_set(instance, (void*) my_data); +/// +/// // You can read your data. +/// { +/// my_data *data = (my_data*) wasmer_instance_context_data_get(wasmer_instance_context_get(instance)); +/// // … +/// } +/// ``` +void wasmer_instance_context_data_set(wasmer_instance_t *instance, + void *data_ptr); + +/// Returns the instance context. Learn more by looking at the +/// `wasmer_instance_context_t` struct. +/// +/// This function returns `null` if `instance` is a null pointer. +/// +/// Example: +/// +/// ```c +/// const wasmer_instance_context_get *context = wasmer_instance_context_get(instance); +/// my_data *data = (my_data *) wasmer_instance_context_data_get(context); +/// // Do something with `my_data`. +/// ``` +/// +/// It is often useful with `wasmer_instance_context_data_set()`. +const wasmer_instance_context_t *wasmer_instance_context_get(wasmer_instance_t *instance); + +/// Gets the `memory_idx`th memory of the instance. +/// +/// Note that the index is always `0` until multiple memories are supported. +/// +/// This function is mostly used inside host functions (aka imported +/// functions) to read the instance memory. +/// +/// Example of a _host function_ that reads and prints a string based on a pointer and a length: +/// +/// ```c +/// void print_string(const wasmer_instance_context_t *context, int32_t pointer, int32_t length) { +/// // Get the 0th memory. +/// const wasmer_memory_t *memory = wasmer_instance_context_memory(context, 0); +/// +/// // Get the memory data as a pointer. +/// uint8_t *memory_bytes = wasmer_memory_data(memory); +/// +/// // Print what we assumed to be a string! +/// printf("%.*s", length, memory_bytes + pointer); +/// } +/// ``` +const wasmer_memory_t *wasmer_instance_context_memory(const wasmer_instance_context_t *ctx, + uint32_t _memory_idx); + +/// Frees memory for the given `wasmer_instance_t`. +/// +/// Check the `wasmer_instantiate()` function to get a complete +/// example. +/// +/// If `instance` is a null pointer, this function does nothing. +/// +/// Example: +/// +/// ```c +/// // Get an instance. +/// wasmer_instance_t *instance = NULL; +/// wasmer_instantiate(&instance, bytes, bytes_length, imports, 0); +/// +/// // Destroy the instance. +/// wasmer_instance_destroy(instance); +/// ``` +void wasmer_instance_destroy(wasmer_instance_t *instance); + +void wasmer_instance_disable_rkyv(); + +void wasmer_instance_enable_rkyv(); + +/// Gets all the exports of the given WebAssembly instance. +/// +/// This function stores a Rust vector of exports into `exports` as an +/// opaque pointer of kind `wasmer_exports_t`. +/// +/// As is, you can do anything with `exports` except using the +/// companion functions, like `wasmer_exports_len()`, +/// `wasmer_exports_get()` or `wasmer_export_kind()`. See the example below. +/// +/// **Warning**: The caller owns the object and should call +/// `wasmer_exports_destroy()` to free it. +/// +/// Example: +/// +/// ```c +/// // Get the exports. +/// wasmer_exports_t *exports = NULL; +/// wasmer_instance_exports(instance, &exports); +/// +/// // Get the number of exports. +/// int exports_length = wasmer_exports_len(exports); +/// printf("Number of exports: %d\n", exports_length); +/// +/// // Read the first export. +/// wasmer_export_t *export = wasmer_exports_get(exports, 0); +/// +/// // Get the kind of the export. +/// wasmer_import_export_kind export_kind = wasmer_export_kind(export); +/// +/// // Assert it is a function (why not). +/// assert(export_kind == WASM_FUNCTION); +/// +/// // Read the export name. +/// wasmer_byte_array name_bytes = wasmer_export_name(export); +/// +/// assert(name_bytes.bytes_len == sizeof("sum") - 1); +/// assert(memcmp(name_bytes.bytes, "sum", sizeof("sum") - 1) == 0); +/// +/// // Destroy the exports. +/// wasmer_exports_destroy(exports); +/// ``` +void wasmer_instance_exports(wasmer_instance_t *instance, wasmer_exports_t **exports); + +wasmer_result_t wasmer_instance_from_cache(wasmer_instance_t **instance, + uint8_t *cache_bytes, + uint32_t cache_len, + const wasmer_compilation_options_t *options); + +uint64_t wasmer_instance_get_points_used(wasmer_instance_t *instance); + +uint64_t wasmer_instance_get_runtime_breakpoint_value(wasmer_instance_t *instance); + +/// Verifies whether the specified function name is imported by the given instance. +bool wasmer_instance_is_function_imported(wasmer_instance_t *instance, const char *name); + +void wasmer_instance_set_points_limit(wasmer_instance_t *instance, uint64_t limit); + +void wasmer_instance_set_points_used(wasmer_instance_t *instance, uint64_t new_gas); + +void wasmer_instance_set_runtime_breakpoint_value(wasmer_instance_t *instance, uint64_t value); + +/// Creates a new WebAssembly instance from the given bytes and imports. +/// +/// The result is stored in the first argument `instance` if +/// successful, i.e. when the function returns +/// `wasmer_result_t::WASMER_OK`. Otherwise +/// `wasmer_result_t::WASMER_ERROR` is returned, and +/// `wasmer_last_error_length()` with `wasmer_last_error_message()` must +/// be used to read the error message. +/// +/// The caller is responsible to free the instance with +/// `wasmer_instance_destroy()`. +/// +/// Example: +/// +/// ```c +/// // 1. Read a WebAssembly module from a file. +/// FILE *file = fopen("sum.wasm", "r"); +/// fseek(file, 0, SEEK_END); +/// long bytes_length = ftell(file); +/// uint8_t *bytes = malloc(bytes_length); +/// fseek(file, 0, SEEK_SET); +/// fread(bytes, 1, bytes_length, file); +/// fclose(file); +/// +/// // 2. Declare the imports (here, none). +/// wasmer_import_t imports[] = {}; +/// +/// // 3. Instantiate the WebAssembly module. +/// wasmer_instance_t *instance = NULL; +/// wasmer_result_t result = wasmer_instantiate(&instance, bytes, bytes_length, imports, 0); +/// +/// // 4. Check for errors. +/// if (result != WASMER_OK) { +/// int error_length = wasmer_last_error_length(); +/// char *error = malloc(error_length); +/// wasmer_last_error_message(error, error_length); +/// // Do something with `error`… +/// } +/// +/// // 5. Free the memory! +/// wasmer_instance_destroy(instance); +/// ``` +wasmer_result_t wasmer_instantiate(wasmer_instance_t **instance, + uint8_t *wasm_bytes, + uint32_t wasm_bytes_len, + wasmer_import_t *imports, + int imports_len); + +wasmer_result_t wasmer_instantiate_with_options(wasmer_instance_t **instance, + uint8_t *wasm_bytes, + uint32_t wasm_bytes_len, + const wasmer_compilation_options_t *options); + +/// Gets the length in bytes of the last error if any. +/// +/// This can be used to dynamically allocate a buffer with the correct number of +/// bytes needed to store a message. +/// +/// See `wasmer_last_error_message()` to get a full example. +int wasmer_last_error_length(); + +/// Gets the last error message if any into the provided buffer +/// `buffer` up to the given `length`. +/// +/// The `length` parameter must be large enough to store the last +/// error message. Ideally, the value should come from +/// `wasmer_last_error_length()`. +/// +/// The function returns the length of the string in bytes, `-1` if an +/// error occurs. Potential errors are: +/// +/// * The buffer is a null pointer, +/// * The buffer is too smal to hold the error message. +/// +/// Note: The error message always has a trailing null character. +/// +/// Example: +/// +/// ```c +/// int error_length = wasmer_last_error_length(); +/// +/// if (error_length > 0) { +/// char *error_message = malloc(error_length); +/// wasmer_last_error_message(error_message, error_length); +/// printf("Error message: `%s`\n", error_message); +/// } else { +/// printf("No error message\n"); +/// } +/// ``` +int wasmer_last_error_message(char *buffer, int length); + +/// Gets a pointer to the beginning of the contiguous memory data +/// bytes. +/// +/// The function returns `NULL` if `memory` is a null pointer. +/// +/// Note that when the memory grows, it can be reallocated, and thus +/// the returned pointer can be invalidated. +/// +/// Example: +/// +/// ```c +/// uint8_t *memory_data = wasmer_memory_data(memory); +/// char *str = (char*) malloc(sizeof(char) * 7); +/// +/// for (uint32_t nth = 0; nth < 7; ++nth) { +/// str[nth] = (char) memory_data[nth]; +/// } +/// ``` +uint8_t *wasmer_memory_data(const wasmer_memory_t *memory); + +/// Gets the size in bytes of the memory data. +/// +/// This function returns 0 if `memory` is a null pointer. +/// +/// Example: +/// +/// ```c +/// uint32_t memory_data_length = wasmer_memory_data_length(memory); +/// ``` +uint32_t wasmer_memory_data_length(wasmer_memory_t *memory); + +/// Frees memory for the given `wasmer_memory_t`. +/// +/// Check the `wasmer_memory_new()` function to get a complete +/// example. +/// +/// If `memory` is a null pointer, this function does nothing. +/// +/// Example: +/// +/// ```c +/// // Get a memory. +/// wasmer_memory_t *memory = NULL; +/// wasmer_result_t result = wasmer_memory_new(&memory, memory_descriptor); +/// +/// // Destroy the memory. +/// wasmer_memory_destroy(memory); +/// ``` +void wasmer_memory_destroy(wasmer_memory_t *memory); + +/// Grows a memory by the given number of pages (of 65Kb each). +/// +/// The functions return `wasmer_result_t::WASMER_OK` upon success, +/// `wasmer_result_t::WASMER_ERROR` otherwise. Use +/// `wasmer_last_error_length()` with `wasmer_last_error_message()` to +/// read the error message. +/// +/// Example: +/// +/// ```c +/// wasmer_result_t result = wasmer_memory_grow(memory, 10); +/// +/// if (result != WASMER_OK) { +/// // … +/// } +/// ``` +wasmer_result_t wasmer_memory_grow(wasmer_memory_t *memory, uint32_t delta); + +/// Reads the current length (in pages) of the given memory. +/// +/// The function returns zero if `memory` is a null pointer. +/// +/// Example: +/// +/// ```c +/// uint32_t memory_length = wasmer_memory_length(memory); +/// +/// printf("Memory pages length: %d\n", memory_length); +/// ``` +uint32_t wasmer_memory_length(const wasmer_memory_t *memory); + +/// Creates a new empty WebAssembly memory for the given descriptor. +/// +/// The result is stored in the first argument `memory` if successful, +/// i.e. when the function returns +/// `wasmer_result_t::WASMER_OK`. Otherwise, +/// `wasmer_result_t::WASMER_ERROR` is returned, and +/// `wasmer_last_error_length()` with `wasmer_last_error_message()` +/// must be used to read the error message. +/// +/// The caller owns the memory and is responsible to free it with +/// `wasmer_memory_destroy()`. +/// +/// Example: +/// +/// ```c +/// // 1. The memory object. +/// wasmer_memory_t *memory = NULL; +/// +/// // 2. The memory descriptor. +/// wasmer_limits_t memory_descriptor = { +/// .min = 10, +/// .max = { +/// .has_some = true, +/// .some = 15, +/// }, +/// }; +/// +/// // 3. Initialize the memory. +/// wasmer_result_t result = wasmer_memory_new(&memory, memory_descriptor); +/// +/// if (result != WASMER_OK) { +/// int error_length = wasmer_last_error_length(); +/// char *error = malloc(error_length); +/// wasmer_last_error_message(error, error_length); +/// // Do something with `error`… +/// } +/// +/// // 4. Free the memory! +/// wasmer_memory_destroy(memory); +/// ``` +wasmer_result_t wasmer_memory_new(wasmer_memory_t **memory, wasmer_limits_t limits); + +/// Deserialize the given serialized module. +/// +/// Returns `wasmer_result_t::WASMER_OK` upon success. +/// +/// Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` +/// and `wasmer_last_error_message` to get an error message. +wasmer_result_t wasmer_module_deserialize(wasmer_module_t **module, + const wasmer_serialized_module_t *serialized_module); + +/// Frees memory for the given Module +void wasmer_module_destroy(wasmer_module_t *module); + +/// Given: +/// * A prepared `wasmer` import-object +/// * A compiled wasmer module +/// +/// Instantiates a wasmer instance +wasmer_result_t wasmer_module_import_instantiate(wasmer_instance_t **instance, + const wasmer_module_t *module, + const wasmer_import_object_t *import_object); + +/// Creates a new Instance from the given module and imports. +/// +/// Returns `wasmer_result_t::WASMER_OK` upon success. +/// +/// Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` +/// and `wasmer_last_error_message` to get an error message. +wasmer_result_t wasmer_module_instantiate(const wasmer_module_t *module, + wasmer_instance_t **instance, + wasmer_import_t *imports, + int imports_len); + +/// Serialize the given Module. +/// +/// The caller owns the object and should call `wasmer_serialized_module_destroy` to free it. +/// +/// Returns `wasmer_result_t::WASMER_OK` upon success. +/// +/// Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` +/// and `wasmer_last_error_message` to get an error message. +wasmer_result_t wasmer_module_serialize(wasmer_serialized_module_t **serialized_module, + const wasmer_module_t *module); + +/// Get bytes of the serialized module. +wasmer_byte_array wasmer_serialized_module_bytes(const wasmer_serialized_module_t *serialized_module); + +/// Frees memory for the given serialized Module. +void wasmer_serialized_module_destroy(wasmer_serialized_module_t *serialized_module); + +/// Transform a sequence of bytes into a serialized module. +/// +/// The caller owns the object and should call `wasmer_serialized_module_destroy` to free it. +/// +/// Returns `wasmer_result_t::WASMER_OK` upon success. +/// +/// Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` +/// and `wasmer_last_error_message` to get an error message. +wasmer_result_t wasmer_serialized_module_from_bytes(wasmer_serialized_module_t **serialized_module, + const uint8_t *serialized_module_bytes, + uint32_t serialized_module_bytes_length); + +void wasmer_set_opcode_costs(const uint32_t *opcode_costs_pointer); + +void wasmer_set_sigsegv_passthrough(); + +/// Frees memory for the given Table +void wasmer_table_destroy(wasmer_table_t *table); + +/// Grows a Table by the given number of elements. +/// +/// Returns `wasmer_result_t::WASMER_OK` upon success. +/// +/// Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` +/// and `wasmer_last_error_message` to get an error message. +wasmer_result_t wasmer_table_grow(wasmer_table_t *table, uint32_t delta); + +/// Returns the current length of the given Table +uint32_t wasmer_table_length(wasmer_table_t *table); + +/// Creates a new Table for the given descriptor and initializes the given +/// pointer to pointer to a pointer to the new Table. +/// +/// The caller owns the object and should call `wasmer_table_destroy` to free it. +/// +/// Returns `wasmer_result_t::WASMER_OK` upon success. +/// +/// Returns `wasmer_result_t::WASMER_ERROR` upon failure. Use `wasmer_last_error_length` +/// and `wasmer_last_error_message` to get an error message. +wasmer_result_t wasmer_table_new(wasmer_table_t **table, wasmer_limits_t limits); + +#if (!defined(_WIN32) && defined(ARCH_X86_64)) +/// Adds a callinfo trampoline to the builder. +uintptr_t wasmer_trampoline_buffer_builder_add_callinfo_trampoline(wasmer_trampoline_buffer_builder_t *builder, + const wasmer_trampoline_callable_t *func, + const void *ctx, + uint32_t num_params); +#endif + +#if (!defined(_WIN32) && defined(ARCH_X86_64)) +/// Adds a context trampoline to the builder. +uintptr_t wasmer_trampoline_buffer_builder_add_context_trampoline(wasmer_trampoline_buffer_builder_t *builder, + const wasmer_trampoline_callable_t *func, + const void *ctx); +#endif + +#if (!defined(_WIN32) && defined(ARCH_X86_64)) +/// Finalizes the trampoline builder into an executable buffer. +wasmer_trampoline_buffer_t *wasmer_trampoline_buffer_builder_build(wasmer_trampoline_buffer_builder_t *builder); +#endif + +#if (!defined(_WIN32) && defined(ARCH_X86_64)) +/// Creates a new trampoline builder. +wasmer_trampoline_buffer_builder_t *wasmer_trampoline_buffer_builder_new(); +#endif + +#if (!defined(_WIN32) && defined(ARCH_X86_64)) +/// Destroys the trampoline buffer if not null. +void wasmer_trampoline_buffer_destroy(wasmer_trampoline_buffer_t *buffer); +#endif + +#if (!defined(_WIN32) && defined(ARCH_X86_64)) +/// Returns the callable pointer for the trampoline with index `idx`. +const wasmer_trampoline_callable_t *wasmer_trampoline_buffer_get_trampoline(const wasmer_trampoline_buffer_t *buffer, + uintptr_t idx); +#endif + +#if (!defined(_WIN32) && defined(ARCH_X86_64)) +/// Returns the context added by `add_context_trampoline`, from within the callee function. +void *wasmer_trampoline_get_context(); +#endif + +/// Stop the execution of a host function, aka imported function. The +/// function must be used _only_ inside a host function. +/// +/// The pointer to `wasmer_instance_context_t` is received by the host +/// function as its first argument. Just passing it to `ctx` is fine. +/// +/// The error message must have a greater lifetime than the host +/// function itself since the error is read outside the host function +/// with `wasmer_last_error_message`. +/// +/// This function returns `wasmer_result_t::WASMER_ERROR` if `ctx` or +/// `error_message` are null. +/// +/// This function never returns otherwise. +wasmer_result_t wasmer_trap(const wasmer_instance_context_t *ctx, const char *error_message); + +/// Validates a sequence of bytes hoping it represents a valid WebAssembly module. +/// +/// The function returns true if the bytes are valid, false otherwise. +/// +/// Example: +/// +/// ```c +/// bool result = wasmer_validate(bytes, bytes_length); +/// +/// if (false == result) { +/// // Do something… +/// } +/// ``` +bool wasmer_validate(const uint8_t *wasm_bytes, uint32_t wasm_bytes_len); + +#if defined(WASMER_WASI_ENABLED) +/// Convenience function that creates a WASI import object with no arguments, +/// environment variables, preopened files, or mapped directories. +/// +/// This function is the same as calling [`wasmer_wasi_generate_import_object`] with all +/// empty values. +wasmer_import_object_t *wasmer_wasi_generate_default_import_object(); +#endif + +#if defined(WASMER_WASI_ENABLED) +/// Creates a WASI import object. +/// +/// This function treats null pointers as empty collections. +/// For example, passing null for a string in `args`, will lead to a zero +/// length argument in that position. +wasmer_import_object_t *wasmer_wasi_generate_import_object(const wasmer_byte_array *args, + unsigned int args_len, + const wasmer_byte_array *envs, + unsigned int envs_len, + const wasmer_byte_array *preopened_files, + unsigned int preopened_files_len, + const wasmer_wasi_map_dir_entry_t *mapped_dirs, + unsigned int mapped_dirs_len); +#endif + +#if defined(WASMER_WASI_ENABLED) +/// Creates a WASI import object for a specific version. +/// +/// This function is similar to `wasmer_wasi_generate_import_object` +/// except that the first argument describes the WASI version. +/// +/// The version is expected to be of kind `Version`. +wasmer_import_object_t *wasmer_wasi_generate_import_object_for_version(unsigned char version, + const wasmer_byte_array *args, + unsigned int args_len, + const wasmer_byte_array *envs, + unsigned int envs_len, + const wasmer_byte_array *preopened_files, + unsigned int preopened_files_len, + const wasmer_wasi_map_dir_entry_t *mapped_dirs, + unsigned int mapped_dirs_len); +#endif + +#if defined(WASMER_WASI_ENABLED) +/// Find the version of WASI used by the module. +/// +/// In case of error, the returned version is `Version::Unknown`. +Version wasmer_wasi_get_version(const wasmer_module_t *module); +#endif + +} // extern "C" + +#endif // WASMER_H diff --git a/lib/runtime-core-tests/Cargo.toml b/lib/runtime-core-tests/Cargo.toml new file mode 100644 index 000000000000..35a6cf5997e4 --- /dev/null +++ b/lib/runtime-core-tests/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "wasmer-runtime-core-tests" +version = "0.15.0" +description = "Tests for the Wasmer runtime core crate" +license = "MIT" +authors = ["The Wasmer Engineering Team "] +edition = "2018" +publish = false + +[dependencies] +wabt = "0.9.1" +wasmer-runtime-core = { path = "../runtime-core", version = "0.15.0" } +wasmer-clif-backend = { path = "../clif-backend", version = "0.15.0", optional = true } +wasmer-singlepass-backend = { path = "../singlepass-backend", version = "0.15.0", optional = true } +wasmer-llvm-backend = { path = "../llvm-backend", version = "0.15.0", features = ["test"], optional = true } + +[features] +default = ["backend-singlepass"] +# backend-cranelift = ["wasmer-clif-backend"] +backend-singlepass = ["wasmer-singlepass-backend"] +backend-llvm = ["wasmer-llvm-backend"] diff --git a/lib/runtime-core/Cargo.toml b/lib/runtime-core/Cargo.toml new file mode 100644 index 000000000000..6bb6477eb122 --- /dev/null +++ b/lib/runtime-core/Cargo.toml @@ -0,0 +1,71 @@ +[package] +name = "wasmer-runtime-core" +version = "0.15.0" +description = "Wasmer runtime core library" +license = "MIT" +authors = ["The Wasmer Engineering Team "] +repository = "https://github.com/ElrondNetwork/wasmer" +keywords = ["wasm", "webassembly", "runtime"] +categories = ["wasm"] +edition = "2018" + +[dependencies] +nix = "0.15" +page_size = "0.4" +wasmparser = { git = "https://github.com/ElrondNetwork/wasmparser.rs" } +parking_lot = "0.10.0" +lazy_static = "1.4" +errno = "0.2" +libc = "0.2.60" +hex = "0.4" +smallvec = "0.6" +bincode = "1.1" +wasm-debug = { optional = true, version = "0.1.0" } +target-lexicon = "0.9" + +[dependencies.rkyv] +version = "0.7.26" +features = ["indexmap"] + +[dependencies.indexmap] +version = "1.7" +features = ["serde-1"] + +# Dependencies for caching. +[dependencies.serde] +version = "1.0" +# This feature is required for serde to support serializing/deserializing reference counted pointers (e.g. Rc and Arc). +features = ["rc"] +[dependencies.serde_derive] +version = "1.0" +[dependencies.serde_bytes] +version = "0.11" +[dependencies.serde-bench] +version = "0.0.7" +[dependencies.blake3] +version = "0.1.0" +[dependencies.digest] +version = "0.8" + +[target.'cfg(windows)'.dependencies] +winapi = { version = "0.3", features = ["memoryapi"] } + +[build-dependencies] +blake3 = "0.1.0" +rustc_version = "0.2" +cc = "1.0" + +[features] +debug = [] +trace = ["debug"] +# backend flags used in conditional compilation of Backend::variants +"backend-cranelift" = [] +"backend-singlepass" = [] +"backend-llvm" = [] +managed = [] +deterministic-execution = ["wasmparser/deterministic"] +# generate debug information from Wasm DWARF for use with the GDB JIT interface +generate-debug-information = ["wasm-debug"] +# don't export symbols related to the GDB JIT interafce, LLVM or some other native +# code will be providing them +generate-debug-information-no-export-symbols = [] diff --git a/lib/runtime-core/src/backend.rs b/lib/runtime-core/src/backend.rs new file mode 100644 index 000000000000..173c0ae36d01 --- /dev/null +++ b/lib/runtime-core/src/backend.rs @@ -0,0 +1,293 @@ +use crate::{ + error::CompileResult, + module::ModuleInner, + state::ModuleStateMap, + typed_func::Wasm, + types::{LocalFuncIndex, SigIndex}, + vm, +}; + +use crate::{ + cache::{Artifact, Error as CacheError}, + codegen::BreakpointMap, + module::ModuleInfo, + sys::Memory, +}; +use std::fmt; +use std::{any::Any, ptr::NonNull}; + +use std::collections::HashMap; + +use rkyv::{ + Archive, + Serialize as RkyvSerialize, + Deserialize as RkyvDeserialize, +}; + +pub mod sys { + pub use crate::sys::*; +} +pub use crate::sig_registry::SigRegistry; + +/// The target architecture for code generation. +#[derive(Copy, Clone, Debug)] +pub enum Architecture { + /// x86-64. + X64, + + /// Aarch64 (ARM64). + Aarch64, +} + +/// The type of an inline breakpoint. +#[repr(u8)] +#[derive(Copy, Clone, Debug)] +pub enum InlineBreakpointType { + /// A middleware invocation breakpoint. + Middleware, +} + +/// Information of an inline breakpoint. +#[derive(Clone, Debug)] +pub struct InlineBreakpoint { + /// Size in bytes taken by this breakpoint's instruction sequence. + pub size: usize, + + /// Type of the inline breakpoint. + pub ty: InlineBreakpointType, +} + +/// This type cannot be constructed from +/// outside the runtime crate. +pub struct Token { + _private: (), +} + +impl Token { + pub(crate) fn generate() -> Self { + Self { _private: () } + } +} + +#[derive(Copy, Clone, Debug)] +pub enum MemoryBoundCheckMode { + Default, + Enable, + Disable, +} + +impl Default for MemoryBoundCheckMode { + fn default() -> MemoryBoundCheckMode { + MemoryBoundCheckMode::Default + } +} + +/// Controls which experimental features will be enabled. +/// Features usually have a corresponding [WebAssembly proposal][wasm-props]. +/// +/// [wasm-props]: https://github.com/WebAssembly/proposals +#[derive(Debug, Default)] +pub struct Features { + /// Whether support for the [SIMD proposal][simd-prop] is enabled. + /// + /// [simd-prop]: https://github.com/webassembly/simd + pub simd: bool, + /// Whether support for the [threads proposal][threads-prop] is enabled. + /// + /// [threads-prop]: https://github.com/webassembly/threads + pub threads: bool, +} + +/// Use this to point to a compiler config struct provided by the backend. +/// The backend struct must support runtime reflection with `Any`, which is any +/// struct that does not contain a non-`'static` reference. +#[derive(Debug)] +pub struct BackendCompilerConfig(pub Box); + +impl BackendCompilerConfig { + /// Obtain the backend-specific compiler config struct. + pub fn get_specific(&self) -> Option<&T> { + self.0.downcast_ref::() + } +} + +/// Configuration data for the compiler +#[derive(Debug, Default)] +pub struct CompilerConfig { + /// Symbol information generated from emscripten; used for more detailed debug messages + pub symbol_map: Option>, + + /// How to make the decision whether to emit bounds checks for memory accesses. + pub memory_bound_check_mode: MemoryBoundCheckMode, + + /// Whether to generate explicit native stack checks against `stack_lower_bound` in `InternalCtx`. + /// + /// Usually it's adequate to use hardware memory protection mechanisms such as `mprotect` on Unix to + /// prevent stack overflow. But for low-level environments, e.g. the kernel, faults are generally + /// not expected and relying on hardware memory protection would add too much complexity. + pub enforce_stack_check: bool, + + /// Whether to enable state tracking. Necessary for managed mode. + pub track_state: bool, + + /// Whether to enable full preemption checkpoint generation. + /// + /// This inserts checkpoints at critical locations such as loop backedges and function calls, + /// allowing preemptive unwinding/task switching. + /// + /// When enabled there can be a small amount of runtime performance overhead. + pub full_preemption: bool, + + pub features: Features, + + // Target info. Presently only supported by LLVM. + pub triple: Option, + pub cpu_name: Option, + pub cpu_features: Option, + + pub backend_specific_config: Option, + + pub generate_debug_info: bool, +} + +impl CompilerConfig { + /// Use this to check if we should be generating debug information. + /// This function takes into account the features that runtime-core was + /// compiled with in addition to the value of the `generate_debug_info` field. + pub(crate) fn should_generate_debug_info(&self) -> bool { + cfg!(feature = "generate-debug-information") && self.generate_debug_info + } +} + +/// An exception table for a `RunnableModule`. +#[derive(Clone, Debug, Default, Serialize, Deserialize, Archive, RkyvSerialize, RkyvDeserialize)] +pub struct ExceptionTable { + /// Mappings from offsets in generated machine code to the corresponding exception code. + pub offset_to_code: HashMap, +} + +impl ExceptionTable { + pub fn new() -> Self { + Self::default() + } +} + +/// The code of an exception. +#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Archive, RkyvSerialize, RkyvDeserialize)] +pub enum ExceptionCode { + /// An `unreachable` opcode was executed. + Unreachable = 0, + /// Call indirect incorrect signature trap. + IncorrectCallIndirectSignature = 1, + /// Memory out of bounds trap. + MemoryOutOfBounds = 2, + /// Call indirect out of bounds trap. + CallIndirectOOB = 3, + /// An arithmetic exception, e.g. divided by zero. + IllegalArithmetic = 4, + /// Misaligned atomic access trap. + MisalignedAtomicAccess = 5, +} + +impl fmt::Display for ExceptionCode { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "{}", + match self { + ExceptionCode::Unreachable => "unreachable", + ExceptionCode::IncorrectCallIndirectSignature => { + "incorrect `call_indirect` signature" + } + ExceptionCode::MemoryOutOfBounds => "memory out-of-bounds access", + ExceptionCode::CallIndirectOOB => "`call_indirect` out-of-bounds", + ExceptionCode::IllegalArithmetic => "illegal arithmetic operation", + ExceptionCode::MisalignedAtomicAccess => "misaligned atomic access", + } + ) + } +} + +pub trait Compiler { + /// Compiles a `Module` from WebAssembly binary format. + /// The `CompileToken` parameter ensures that this can only + /// be called from inside the runtime. + fn compile( + &self, + wasm: &[u8], + comp_conf: CompilerConfig, + _: Token, + ) -> CompileResult; + + unsafe fn from_cache(&self, cache: Artifact, _: Token) -> Result; +} + +pub trait RunnableModule: Send + Sync { + /// This returns a pointer to the function designated by the `local_func_index` + /// parameter. + fn get_func( + &self, + info: &ModuleInfo, + local_func_index: LocalFuncIndex, + ) -> Option>; + + fn get_module_state_map(&self) -> Option { + None + } + + fn get_breakpoints(&self) -> Option { + None + } + + fn get_exception_table(&self) -> Option<&ExceptionTable> { + None + } + + unsafe fn patch_local_function(&self, _idx: usize, _target_address: usize) -> bool { + false + } + + /// A wasm trampoline contains the necessary data to dynamically call an exported wasm function. + /// Given a particular signature index, we return a trampoline that is matched with that + /// signature and an invoke function that can call the trampoline. + fn get_trampoline(&self, info: &ModuleInfo, sig_index: SigIndex) -> Option; + + /// Trap an error. + unsafe fn do_early_trap(&self, data: Box) -> !; + + /// Returns the machine code associated with this module. + fn get_code(&self) -> Option<&[u8]> { + None + } + + /// Returns the beginning offsets of all functions, including import trampolines. + fn get_offsets(&self) -> Option> { + None + } + + /// Returns the beginning offsets of all local functions. + fn get_local_function_offsets(&self) -> Option> { + None + } + + /// Returns the inline breakpoint size corresponding to an Architecture (None in case is not implemented) + fn get_inline_breakpoint_size(&self, _arch: Architecture) -> Option { + None + } + + /// Attempts to read an inline breakpoint from the code. + /// + /// Inline breakpoints are detected by special instruction sequences that never + /// appear in valid code. + fn read_inline_breakpoint( + &self, + _arch: Architecture, + _code: &[u8], + ) -> Option { + None + } +} + +pub trait CacheGen: Send + Sync { + fn generate_cache(&self) -> Result<(Box<[u8]>, Memory), CacheError>; +} diff --git a/lib/runtime-core/src/cache.rs b/lib/runtime-core/src/cache.rs new file mode 100644 index 000000000000..3f8820788341 --- /dev/null +++ b/lib/runtime-core/src/cache.rs @@ -0,0 +1,346 @@ +//! The cache module provides the common data structures used by compiler backends to allow +//! serializing compiled wasm code to a binary format. The binary format can be persisted, +//! and loaded to allow skipping compilation and fast startup. + +use crate::{module::ModuleInfo, sys::Memory, sys::ArchivableMemory}; +use rkyv::{Archive, Serialize as RkyvSerialize, Deserialize as RkyvDeserialize}; +use std::{io, mem, slice}; + +/// Indicates the invalid type of invalid cache file +#[derive(Debug)] +pub enum InvalidFileType { + /// Given cache header slice does not match the expected size of an `ArtifactHeader` + InvalidSize, + /// Given cache header slice does not contain the expected magic bytes + InvalidMagic, +} + +/// Kinds of caching errors +#[derive(Debug)] +pub enum Error { + /// An IO error while reading/writing a cache binary. + IoError(io::Error), + /// An error deserializing bytes into a cache data structure. + DeserializeError(String), + /// An error serializing bytes from a cache data structure. + SerializeError(String), + /// An undefined caching error with a message. + Unknown(String), + /// An invalid cache binary given. + InvalidFile(InvalidFileType), + /// The cached binary has been invalidated. + InvalidatedCache, + /// The current backend does not support caching. + UnsupportedBackend(String), +} + +impl From for Error { + fn from(io_err: io::Error) -> Self { + Error::IoError(io_err) + } +} + +/// The hash of a wasm module. +/// +/// Used as a key when loading and storing modules in a [`Cache`]. +/// +/// [`Cache`]: trait.Cache.html +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +// WasmHash is made up of a 32 byte array +pub struct WasmHash([u8; 32]); + +impl WasmHash { + /// Hash a wasm module. + /// + /// # Note: + /// This does no verification that the supplied data + /// is, in fact, a wasm module. + pub fn generate(wasm: &[u8]) -> Self { + let hash = blake3::hash(wasm); + WasmHash(hash.into()) + } + + /// Create the hexadecimal representation of the + /// stored hash. + pub fn encode(self) -> String { + hex::encode(&self.into_array() as &[u8]) + } + + /// Create hash from hexadecimal representation + pub fn decode(hex_str: &str) -> Result { + let bytes = hex::decode(hex_str).map_err(|e| { + Error::DeserializeError(format!( + "Could not decode prehashed key as hexadecimal: {}", + e + )) + })?; + if bytes.len() != 32 { + return Err(Error::DeserializeError( + "Prehashed keys must deserialze into exactly 32 bytes".to_string(), + )); + } + use std::convert::TryInto; + Ok(WasmHash(bytes[0..32].try_into().map_err(|e| { + Error::DeserializeError(format!("Could not get first 32 bytes: {}", e)) + })?)) + } + + pub(crate) fn into_array(self) -> [u8; 32] { + let mut total = [0u8; 32]; + total[0..32].copy_from_slice(&self.0); + total + } +} + +const CURRENT_CACHE_VERSION: u64 = 0; +static WASMER_CACHE_MAGIC: [u8; 8] = *b"WASMER\0\0"; + +/// The header of a cache file. +#[repr(C, packed)] +struct ArtifactHeader { + magic: [u8; 8], // [W, A, S, M, E, R, \0, \0] + version: u64, + data_len: u64, +} + +impl ArtifactHeader { + pub fn read_from_slice(buffer: &[u8]) -> Result<(&Self, &[u8]), Error> { + if buffer.len() >= mem::size_of::() { + if &buffer[..8] == &WASMER_CACHE_MAGIC { + let (header_slice, body_slice) = buffer.split_at(mem::size_of::()); + let header = unsafe { &*(header_slice.as_ptr() as *const ArtifactHeader) }; + + if header.version == CURRENT_CACHE_VERSION { + Ok((header, body_slice)) + } else { + println!("invalidated cache"); + Err(Error::InvalidatedCache) + } + } else { + println!("invalid magic"); + Err(Error::InvalidFile(InvalidFileType::InvalidMagic)) + } + } else { + println!("invalid size"); + Err(Error::InvalidFile(InvalidFileType::InvalidSize)) + } + } + + pub fn read_from_slice_mut(buffer: &mut [u8]) -> Result<(&mut Self, &mut [u8]), Error> { + if buffer.len() >= mem::size_of::() { + if &buffer[..8] == &WASMER_CACHE_MAGIC { + let (header_slice, body_slice) = + buffer.split_at_mut(mem::size_of::()); + let header = unsafe { &mut *(header_slice.as_ptr() as *mut ArtifactHeader) }; + + if header.version == CURRENT_CACHE_VERSION { + Ok((header, body_slice)) + } else { + Err(Error::InvalidatedCache) + } + } else { + Err(Error::InvalidFile(InvalidFileType::InvalidMagic)) + } + } else { + Err(Error::InvalidFile(InvalidFileType::InvalidSize)) + } + } + + pub fn as_slice(&self) -> &[u8] { + let ptr = self as *const ArtifactHeader as *const u8; + unsafe { slice::from_raw_parts(ptr, mem::size_of::()) } + } +} + + +/// Inner information of an Artifact. +#[derive(Serialize, Deserialize, Archive, RkyvSerialize, RkyvDeserialize)] +pub struct ArtifactInner { + info: Box, + #[serde(with = "serde_bytes")] + backend_metadata: Box<[u8]>, + + #[with(ArchivableMemory)] + compiled_code: Memory, +} + +/// Artifact are produced by caching, are serialized/deserialized to binaries, and contain +/// module info, backend metadata, and compiled code. +#[derive(Archive, RkyvSerialize, RkyvDeserialize)] +pub struct Artifact { + inner: ArtifactInner, +} + +impl Artifact { + pub(crate) fn from_parts( + info: Box, + backend_metadata: Box<[u8]>, + compiled_code: Memory, + ) -> Self { + Self { + inner: ArtifactInner { + info, + backend_metadata, + compiled_code, + }, + } + } + + /// A reference to the `Artifact`'s stored `ModuleInfo` + pub fn info(&self) -> &ModuleInfo { + &self.inner.info + } + + #[doc(hidden)] + pub fn consume(self) -> (ModuleInfo, Box<[u8]>, Memory) { + ( + *self.inner.info, + self.inner.backend_metadata, + self.inner.compiled_code, + ) + } + + /// Deserializes an `Artifact` from the given byte slice. + pub fn deserialize(bytes: &[u8]) -> Result { + let (_, body_slice) = ArtifactHeader::read_from_slice(bytes)?; + + let inner = serde_bench::deserialize(body_slice) + .map_err(|e| Error::DeserializeError(format!("{:#?}", e)))?; + + Ok(Artifact { inner }) + } + + /// Serializes the `Artifact` into a vector of bytes + pub fn serialize(&self) -> Result, Error> { + let cache_header = ArtifactHeader { + magic: WASMER_CACHE_MAGIC, + version: CURRENT_CACHE_VERSION, + data_len: 0, + }; + + let mut buffer = cache_header.as_slice().to_vec(); + + serde_bench::serialize(&mut buffer, &self.inner) + .map_err(|e| Error::SerializeError(e.to_string()))?; + + let data_len = (buffer.len() - mem::size_of::()) as u64; + let (header, _) = ArtifactHeader::read_from_slice_mut(&mut buffer)?; + header.data_len = data_len; + + Ok(buffer) + } +} + +/// A unique ID generated from the version of Wasmer for use with cache versioning +pub const WASMER_VERSION_HASH: &'static str = + include_str!(concat!(env!("OUT_DIR"), "/wasmer_version_hash.txt")); + +#[cfg(test)] +mod tests { + use super::Artifact; + use super::ArtifactInner; + use super::Memory; + use super::ModuleInfo; + use std::collections::HashMap; + use crate::structures::Map; + use crate::module::StringTable; + use rkyv::ser::serializers::AllocSerializer; + use rkyv::ser::Serializer as RkyvSerializer; + use rkyv::Deserialize; + use rkyv::Archived; + + #[test] + fn test_rkyv_artifact() { + let bytes = make_test_bytes(); + let memory = make_test_memory(&bytes); + + let module_info = make_empty_module_info(); + let artifact = Artifact::from_parts( + Box::new(module_info), + b"test_backend".to_vec().into_boxed_slice(), + memory, + ); + + let mut serializer = AllocSerializer::<4096>::default(); + serializer.serialize_value(&artifact).unwrap(); + let serialized = serializer.into_serializer().into_inner(); + assert!(serialized.len() > 0); + print!("{:?}", serialized); + + let archived: &Archived + = unsafe { rkyv::archived_root::(&serialized[..]) }; + + let deserialized_artifact = Deserialize::::deserialize(archived, &mut rkyv::Infallible).unwrap(); + unsafe { assert_eq!(deserialized_artifact.inner.compiled_code.as_slice(), artifact.inner.compiled_code.as_slice()) }; + assert_eq!(deserialized_artifact.inner.compiled_code.protection(), artifact.inner.compiled_code.protection()); + } + + #[test] + fn test_rkyv_artifact_inner() { + let bytes = make_test_bytes(); + let memory = make_test_memory(&bytes); + + let module_info = make_empty_module_info(); + let artifact_inner = ArtifactInner { + info: Box::new(module_info), + backend_metadata: b"test_backend".to_vec().into_boxed_slice(), + compiled_code: memory, + }; + + let mut serializer = AllocSerializer::<4096>::default(); + serializer.serialize_value(&artifact_inner).unwrap(); + let serialized = serializer.into_serializer().into_inner(); + assert!(serialized.len() > 0); + print!("{:?}", serialized); + + let archived: &Archived + = unsafe { rkyv::archived_root::(&serialized[..]) }; + + let deserialized_artifact_inner = Deserialize::::deserialize(archived, &mut rkyv::Infallible).unwrap(); + unsafe { assert_eq!(deserialized_artifact_inner.compiled_code.as_slice(), artifact_inner.compiled_code.as_slice()) }; + assert_eq!(deserialized_artifact_inner.compiled_code.protection(), artifact_inner.compiled_code.protection()); + } + + fn make_empty_module_info() -> ModuleInfo { + ModuleInfo { + memories: Map::new(), + globals: Map::new(), + tables: Map::new(), + imported_functions: Map::new(), + imported_memories: Map::new(), + imported_tables: Map::new(), + imported_globals: Map::new(), + exports: Default::default(), + data_initializers: Vec::new(), + elem_initializers: Vec::new(), + start_func: None, + func_assoc: Map::new(), + signatures: Map::new(), + backend: "test".to_string(), + namespace_table: StringTable::new(), + name_table: StringTable::new(), + em_symbol_map: None, + custom_sections: HashMap::new(), + generate_debug_info: false, + #[cfg(feature = "generate-debug-information")] + debug_info_manager: crate::jit_debug::JitCodeDebugInfoManager::new(), + } + } + + fn make_test_memory(bytes: &Vec) -> Memory { + let mut memory = Memory::with_size_protect(1000, crate::sys::Protect::ReadWrite) + .expect("Could not create memory"); + unsafe { + memory.as_slice_mut().copy_from_slice(&bytes[..]); + } + memory + } + + fn make_test_bytes() -> Vec { + let page_size = page_size::get(); + let mut bytes = b"abcdefghijkl".to_vec(); + let padding_zeros = [0 as u8; 1].repeat(page_size - bytes.len()); + bytes.extend(padding_zeros); + bytes + } +} diff --git a/lib/runtime-core/src/codegen.rs b/lib/runtime-core/src/codegen.rs new file mode 100644 index 000000000000..bf035aee3b6e --- /dev/null +++ b/lib/runtime-core/src/codegen.rs @@ -0,0 +1,528 @@ +//! The codegen module provides common functions and data structures used by multiple backends +//! during the code generation process. +#[cfg(unix)] +use crate::fault::FaultInfo; +use crate::{ + backend::RunnableModule, + backend::{CacheGen, Compiler, CompilerConfig, Features, Token}, + cache::{Artifact, Error as CacheError}, + error::{CompileError, CompileResult}, + module::{ModuleInfo, ModuleInner}, + structures::Map, + types::{FuncIndex, FuncSig, SigIndex}, +}; +use smallvec::SmallVec; +use std::any::Any; +use std::collections::HashMap; +use std::fmt; +use std::fmt::Debug; +use std::marker::PhantomData; +use std::sync::{Arc, RwLock}; +use wasmparser::{self, WasmDecoder}; +use wasmparser::{Operator, Type as WpType}; + +/// A type that defines a function pointer, which is called when breakpoints occur. +pub type BreakpointHandler = + Box Result<(), Box> + Send + Sync + 'static>; + +/// Maps instruction pointers to their breakpoint handlers. +pub type BreakpointMap = Arc>; + +/// An event generated during parsing of a wasm binary +#[derive(Debug)] +pub enum Event<'a, 'b> { + /// An internal event created by the parser used to provide hooks during code generation. + Internal(InternalEvent), + /// An event generated by parsing a wasm operator + Wasm(&'b Operator<'a>), + /// An event generated by parsing a wasm operator that contains an owned `Operator` + WasmOwned(Operator<'a>), +} + +/// Kinds of `InternalEvent`s created during parsing. +pub enum InternalEvent { + /// A function parse is about to begin. + FunctionBegin(u32), + /// A function parsing has just completed. + FunctionEnd, + /// A breakpoint emitted during parsing. + Breakpoint(BreakpointHandler), + /// Indicates setting an internal field. + SetInternal(u32), + /// Indicates getting an internal field. + GetInternal(u32), +} + +impl fmt::Debug for InternalEvent { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + InternalEvent::FunctionBegin(_) => write!(f, "FunctionBegin"), + InternalEvent::FunctionEnd => write!(f, "FunctionEnd"), + InternalEvent::Breakpoint(_) => write!(f, "Breakpoint"), + InternalEvent::SetInternal(_) => write!(f, "SetInternal"), + InternalEvent::GetInternal(_) => write!(f, "GetInternal"), + } + } +} + +/// Type representing an area of Wasm code in bytes as an offset from the +/// beginning of the code section. +/// +/// `start` must be less than or equal to `end`. +#[derive(Copy, Clone, Debug)] +pub struct WasmSpan { + /// Start offset in bytes from the beginning of the Wasm code section + start: u32, + /// End offset in bytes from the beginning of the Wasm code section + end: u32, +} + +impl WasmSpan { + /// Create a new `WasmSpan`. + /// + /// `start` must be less than or equal to `end`. + // TODO: mark this function as `const` when asserts get stabilized as `const` + // see: https://github.com/rust-lang/rust/issues/57563 + pub fn new(start: u32, end: u32) -> Self { + debug_assert!(start <= end); + Self { start, end } + } + + /// Start offset in bytes from the beginning of the Wasm code section + pub const fn start(&self) -> u32 { + self.start + } + + /// End offset in bytes from the beginning of the Wasm code section + pub const fn end(&self) -> u32 { + self.end + } + + /// Size in bytes of the span + pub const fn size(&self) -> u32 { + self.end - self.start + } +} + +/// Information for a breakpoint +#[cfg(unix)] +pub struct BreakpointInfo<'a> { + /// Fault. + pub fault: Option<&'a FaultInfo>, +} + +/// Information for a breakpoint +#[cfg(not(unix))] +pub struct BreakpointInfo { + /// Fault placeholder. + pub fault: Option<()>, +} + +/// A trait that represents the functions needed to be implemented to generate code for a module. +pub trait ModuleCodeGenerator, RM: RunnableModule, E: Debug> { + /// Creates a new module code generator. + fn new() -> Self; + + /// Creates a new module code generator for specified target. + fn new_with_target( + triple: Option, + cpu_name: Option, + cpu_features: Option, + ) -> Self; + + /// Returns the backend id associated with this MCG. + fn backend_id() -> &'static str; + + /// It sets if the current compiler requires validation before compilation + fn requires_pre_validation() -> bool { + true + } + + /// Feeds the compiler config. + fn feed_compiler_config(&mut self, _config: &CompilerConfig) -> Result<(), E> { + Ok(()) + } + /// Adds an import function. + fn feed_import_function(&mut self) -> Result<(), E>; + /// Sets the signatures. + fn feed_signatures(&mut self, signatures: Map) -> Result<(), E>; + /// Sets function signatures. + fn feed_function_signatures(&mut self, assoc: Map) -> Result<(), E>; + /// Checks the precondition for a module. + fn check_precondition(&mut self, module_info: &ModuleInfo) -> Result<(), E>; + /// Creates a new function and returns the function-scope code generator for it. + fn next_function( + &mut self, + module_info: Arc>, + loc: WasmSpan, + ) -> Result<&mut FCG, E>; + /// Finalizes this module. + fn finalize( + self, + module_info: &ModuleInfo, + ) -> Result<(RM, Option, Box), E>; + + /// Creates a module from cache. + unsafe fn from_cache(cache: Artifact, _: Token) -> Result; +} + +/// Mock item when compiling without debug info generation. +#[cfg(not(feature = "generate-debug-information"))] +type CompiledFunctionData = (); + +/// Mock item when compiling without debug info generation. +#[cfg(not(feature = "generate-debug-information"))] +type ValueLabelsRangesInner = (); + +#[cfg(feature = "generate-debug-information")] +use wasm_debug::types::{CompiledFunctionData, ValueLabelsRangesInner}; + +#[derive(Clone, Debug)] +/// Useful information for debugging gathered by compiling a Wasm module. +pub struct DebugMetadata { + /// [`CompiledFunctionData`] in [`FuncIndex`] order + pub func_info: Map, + /// [`ValueLabelsRangesInner`] in [`FuncIndex`] order + pub inst_info: Map, + /// Stack slot offsets in [`FuncIndex`] order + pub stack_slot_offsets: Map>>, + /// function pointers and their lengths + pub pointers: Vec<(*const u8, usize)>, +} + +/// A streaming compiler which is designed to generated code for a module based on a stream +/// of wasm parser events. +pub struct StreamingCompiler< + MCG: ModuleCodeGenerator, + FCG: FunctionCodeGenerator, + RM: RunnableModule + 'static, + E: Debug, + CGEN: Fn() -> MiddlewareChain, +> { + middleware_chain_generator: CGEN, + _phantom_mcg: PhantomData, + _phantom_fcg: PhantomData, + _phantom_rm: PhantomData, + _phantom_e: PhantomData, +} + +/// A simple generator for a `StreamingCompiler`. +pub struct SimpleStreamingCompilerGen< + MCG: ModuleCodeGenerator, + FCG: FunctionCodeGenerator, + RM: RunnableModule + 'static, + E: Debug, +> { + _phantom_mcg: PhantomData, + _phantom_fcg: PhantomData, + _phantom_rm: PhantomData, + _phantom_e: PhantomData, +} + +impl< + MCG: ModuleCodeGenerator, + FCG: FunctionCodeGenerator, + RM: RunnableModule + 'static, + E: Debug, + > SimpleStreamingCompilerGen +{ + /// Create a new `StreamingCompiler`. + pub fn new() -> StreamingCompiler MiddlewareChain> { + StreamingCompiler::new(|| MiddlewareChain::new()) + } +} + +impl< + MCG: ModuleCodeGenerator, + FCG: FunctionCodeGenerator, + RM: RunnableModule + 'static, + E: Debug, + CGEN: Fn() -> MiddlewareChain, + > StreamingCompiler +{ + /// Create a new `StreamingCompiler` with the given `MiddlewareChain`. + pub fn new(chain_gen: CGEN) -> Self { + Self { + middleware_chain_generator: chain_gen, + _phantom_mcg: PhantomData, + _phantom_fcg: PhantomData, + _phantom_rm: PhantomData, + _phantom_e: PhantomData, + } + } +} + +/// Create a new `ValidatingParserConfig` with the given features. +pub fn validating_parser_config(features: &Features) -> wasmparser::ValidatingParserConfig { + wasmparser::ValidatingParserConfig { + operator_config: wasmparser::OperatorValidatorConfig { + enable_threads: features.threads, + enable_reference_types: false, + enable_simd: features.simd, + enable_bulk_memory: false, + enable_multi_value: false, + + #[cfg(feature = "deterministic-execution")] + deterministic_only: true, + }, + } +} + +fn validate_with_features(bytes: &[u8], features: &Features) -> CompileResult<()> { + let mut parser = + wasmparser::ValidatingParser::new(bytes, Some(validating_parser_config(features))); + loop { + let state = parser.read(); + match *state { + wasmparser::ParserState::EndWasm => break Ok(()), + wasmparser::ParserState::Error(ref err) => Err(CompileError::ValidationError { + msg: err.message().to_string(), + })?, + _ => {} + } + } +} + +impl< + MCG: ModuleCodeGenerator, + FCG: FunctionCodeGenerator, + RM: RunnableModule + 'static, + E: Debug, + CGEN: Fn() -> MiddlewareChain, + > Compiler for StreamingCompiler +{ + #[allow(unused_variables)] + fn compile( + &self, + wasm: &[u8], + compiler_config: CompilerConfig, + _: Token, + ) -> CompileResult { + if MCG::requires_pre_validation() { + validate_with_features(wasm, &compiler_config.features)?; + } + + let mut mcg = match MCG::backend_id() { + "llvm" => MCG::new_with_target( + compiler_config.triple.clone(), + compiler_config.cpu_name.clone(), + compiler_config.cpu_features.clone(), + ), + _ => MCG::new(), + }; + let mut chain = (self.middleware_chain_generator)(); + let info = crate::parse::read_module(wasm, &mut mcg, &mut chain, &compiler_config)?; + let (exec_context, compile_debug_info, cache_gen) = mcg + .finalize(&info.read().unwrap()) + .map_err(|x| CompileError::InternalError { + msg: format!("{:?}", x), + })?; + + #[cfg(feature = "generate-debug-information")] + { + if compiler_config.should_generate_debug_info() { + if let Some(dbg_info) = compile_debug_info { + let debug_info = wasm_debug::read_debuginfo(wasm); + let extra_info = wasm_debug::types::ModuleVmctxInfo::new( + crate::vm::Ctx::offset_memory_base() as _, + std::mem::size_of::() as _, + dbg_info.stack_slot_offsets.values(), + ); + let compiled_fn_map = + wasm_debug::types::create_module_address_map(dbg_info.func_info.values()); + let range_map = + wasm_debug::types::build_values_ranges(dbg_info.inst_info.values()); + let raw_func_slice = &dbg_info.pointers; + + let debug_image = wasm_debug::emit_debugsections_image( + target_lexicon::HOST, + std::mem::size_of::() as u8, + &debug_info, + &extra_info, + &compiled_fn_map, + &range_map, + raw_func_slice, + ) + .expect("make debug image"); + + let mut writer = info.write().unwrap(); + writer + .debug_info_manager + .register_new_jit_code_entry(&debug_image); + } + } + } + + Ok(ModuleInner { + cache_gen, + runnable_module: Arc::new(Box::new(exec_context)), + info: Arc::try_unwrap(info).unwrap().into_inner().unwrap(), + }) + } + + unsafe fn from_cache( + &self, + artifact: Artifact, + token: Token, + ) -> Result { + MCG::from_cache(artifact, token) + } +} + +/// A sink for parse events. +pub struct EventSink<'a, 'b> { + buffer: SmallVec<[Event<'a, 'b>; 2]>, +} + +impl<'a, 'b> EventSink<'a, 'b> { + /// Push a new `Event` to this sink. + pub fn push(&mut self, ev: Event<'a, 'b>) { + self.buffer.push(ev); + } +} + +/// A container for a chain of middlewares. +pub struct MiddlewareChain { + chain: Vec>, +} + +impl MiddlewareChain { + /// Create a new empty `MiddlewareChain`. + pub fn new() -> MiddlewareChain { + MiddlewareChain { chain: vec![] } + } + + /// Push a new `FunctionMiddleware` to this `MiddlewareChain`. + pub fn push(&mut self, m: M) { + self.chain.push(Box::new(m)); + } + + /// Run this chain with the provided function code generator, event and module info. + pub(crate) fn run>( + &mut self, + fcg: Option<&mut FCG>, + ev: Event, + module_info: &ModuleInfo, + source_loc: u32, + ) -> Result<(), String> { + let mut sink = EventSink { + buffer: SmallVec::new(), + }; + sink.push(ev); + for m in &mut self.chain { + let prev: SmallVec<[Event; 2]> = sink.buffer.drain().collect(); + for ev in prev { + m.feed_event(ev, module_info, &mut sink, source_loc)?; + } + } + if let Some(fcg) = fcg { + for ev in sink.buffer { + fcg.feed_event(ev, module_info, source_loc) + .map_err(|x| format!("{:?}", x))?; + } + } + + Ok(()) + } + + /// Notify this chain about a given local variable. + pub(crate) fn run_func_local( + &mut self, + ty: WpType, + n: usize, + loc: u32, + ) -> Result<(), String> { + for m in &mut self.chain { + m.feed_local(ty, n, loc) + .map_err(|x| format!("{:?}", x))?; + } + + Ok(()) + } +} + +/// A trait that represents the signature required to implement middleware for a function. +pub trait FunctionMiddleware { + /// The error type for this middleware's functions. + type Error: Debug; + /// Processes the given event, module info and sink. + fn feed_event<'a, 'b: 'a>( + &mut self, + op: Event<'a, 'b>, + module_info: &ModuleInfo, + sink: &mut EventSink<'a, 'b>, + source_loc: u32, + ) -> Result<(), Self::Error>; + + /// Notify the middleware about a given local variable. + fn feed_local( + &mut self, + _ty: WpType, + _n: usize, + _source_loc: u32, + ) -> Result<(), Self::Error> { + Ok(()) + } +} + +pub(crate) trait GenericFunctionMiddleware { + fn feed_event<'a, 'b: 'a>( + &mut self, + op: Event<'a, 'b>, + module_info: &ModuleInfo, + sink: &mut EventSink<'a, 'b>, + source_loc: u32, + ) -> Result<(), String>; + + fn feed_local( + &mut self, + _ty: WpType, + _n: usize, + _source_loc: u32, + ) -> Result<(), String>; +} + +impl> GenericFunctionMiddleware for T { + fn feed_event<'a, 'b: 'a>( + &mut self, + op: Event<'a, 'b>, + module_info: &ModuleInfo, + sink: &mut EventSink<'a, 'b>, + source_loc: u32, + ) -> Result<(), String> { + ::feed_event(self, op, module_info, sink, source_loc) + .map_err(|x| format!("{:?}", x)) + } + + fn feed_local( + &mut self, + ty: WpType, + n: usize, + source_loc: u32, + ) -> Result<(), String> { + ::feed_local(self, ty, n, source_loc) + .map_err(|x| format!("{:?}", x)) + } +} + +/// The function-scope code generator trait. +pub trait FunctionCodeGenerator { + /// Sets the return type. + fn feed_return(&mut self, ty: WpType) -> Result<(), E>; + + /// Adds a parameter to the function. + fn feed_param(&mut self, ty: WpType) -> Result<(), E>; + + /// Adds `n` locals to the function. + fn feed_local(&mut self, ty: WpType, n: usize, loc: u32) -> Result<(), E>; + + /// Called before the first call to `feed_opcode`. + fn begin_body(&mut self, module_info: &ModuleInfo) -> Result<(), E>; + + /// Called for each operator. + fn feed_event(&mut self, op: Event, module_info: &ModuleInfo, source_loc: u32) + -> Result<(), E>; + + /// Finalizes the function. + fn finalize(&mut self) -> Result<(), E>; +} diff --git a/lib/runtime-core/src/fault.rs b/lib/runtime-core/src/fault.rs new file mode 100644 index 000000000000..5bfd6e7b63cb --- /dev/null +++ b/lib/runtime-core/src/fault.rs @@ -0,0 +1,1048 @@ +//! The fault module contains the implementation for handling breakpoints, traps, and signals +//! for wasm code. + +pub mod raw { + //! The raw module contains required externed function interfaces for the fault module. + use std::ffi::c_void; + + #[cfg(target_arch = "x86_64")] + extern "C" { + /// Load registers and return on the stack [stack_end..stack_begin]. + pub fn run_on_alternative_stack(stack_end: *mut u64, stack_begin: *mut u64) -> u64; + /// Internal routine for switching into a backend without information about where registers are preserved. + pub fn register_preservation_trampoline(); // NOT safe to call directly + } + + /// Internal routine for switching into a backend without information about where registers are preserved. + #[cfg(not(target_arch = "x86_64"))] + pub extern "C" fn register_preservation_trampoline() { + unimplemented!("register_preservation_trampoline"); + } + + extern "C" { + /// libc setjmp + pub fn setjmp(env: *mut c_void) -> i32; + /// libc longjmp + pub fn longjmp(env: *mut c_void, val: i32) -> !; + } +} + +use crate::codegen::{BreakpointInfo, BreakpointMap}; +use crate::state::x64::{build_instance_image, read_stack, X64Register, GPR}; +use crate::state::{CodeVersion, ExecutionStateImage}; +use crate::vm; +use libc::{mmap, mprotect, siginfo_t, MAP_ANON, MAP_PRIVATE, PROT_NONE, PROT_READ, PROT_WRITE}; +use nix::sys::signal::{ + sigaction, SaFlags, SigAction, SigHandler, SigSet, Signal, SIGBUS, SIGFPE, SIGILL, SIGINT, + SIGSEGV, SIGTRAP, +}; +use std::any::Any; +use std::cell::{Cell, RefCell, UnsafeCell}; +use std::ffi::c_void; +use std::process; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Once; + +#[cfg(target_arch = "x86_64")] +pub(crate) unsafe fn run_on_alternative_stack(stack_end: *mut u64, stack_begin: *mut u64) -> u64 { + raw::run_on_alternative_stack(stack_end, stack_begin) +} + +#[cfg(not(target_arch = "x86_64"))] +pub(crate) unsafe fn run_on_alternative_stack(_stack_end: *mut u64, _stack_begin: *mut u64) -> u64 { + unimplemented!("run_on_alternative_stack"); +} + +const TRAP_STACK_SIZE: usize = 1048576; // 1MB + +const SETJMP_BUFFER_LEN: usize = 128; +type SetJmpBuffer = [i32; SETJMP_BUFFER_LEN]; + +struct UnwindInfo { + jmpbuf: SetJmpBuffer, // in + breakpoints: Option, + payload: Option>, // out +} + +/// A store for boundary register preservation. +#[repr(packed)] +#[derive(Default, Copy, Clone)] +pub struct BoundaryRegisterPreservation { + /// R15. + pub r15: u64, + /// R14. + pub r14: u64, + /// R13. + pub r13: u64, + /// R12. + pub r12: u64, + /// RBX. + pub rbx: u64, +} + +thread_local! { + static UNWIND: UnsafeCell> = UnsafeCell::new(None); + static CURRENT_CTX: UnsafeCell<*mut vm::Ctx> = UnsafeCell::new(::std::ptr::null_mut()); + static CURRENT_CODE_VERSIONS: RefCell> = RefCell::new(vec![]); + static WAS_SIGINT_TRIGGERED: Cell = Cell::new(false); + static BOUNDARY_REGISTER_PRESERVATION: UnsafeCell = UnsafeCell::new(BoundaryRegisterPreservation::default()); +} + +/// Gets a mutable pointer to the `BoundaryRegisterPreservation`. +#[no_mangle] +pub unsafe extern "C" fn get_boundary_register_preservation() -> *mut BoundaryRegisterPreservation { + BOUNDARY_REGISTER_PRESERVATION.with(|x| x.get()) +} + +struct InterruptSignalMem(*mut u8); +unsafe impl Send for InterruptSignalMem {} +unsafe impl Sync for InterruptSignalMem {} + +const INTERRUPT_SIGNAL_MEM_SIZE: usize = 4096; + +lazy_static! { + static ref INTERRUPT_SIGNAL_MEM: InterruptSignalMem = { + let ptr = unsafe { + mmap( + ::std::ptr::null_mut(), + INTERRUPT_SIGNAL_MEM_SIZE, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON, + -1, + 0, + ) + }; + if ptr as isize == -1 { + panic!("cannot allocate code memory"); + } + InterruptSignalMem(ptr as _) + }; +} + +static INTERRUPT_SIGNAL_DELIVERED: AtomicBool = AtomicBool::new(false); + +/// Controls whether SIGSEGV is handled by Wasmer or not. +pub static SIGSEGV_PASSTHROUGH: AtomicBool = AtomicBool::new(false); + +/// Returns a boolean indicating if SIGINT triggered the fault. +pub fn was_sigint_triggered_fault() -> bool { + WAS_SIGINT_TRIGGERED.with(|x| x.get()) +} + +/// Runs a callback function with the given `Ctx`. +pub unsafe fn with_ctx R>(ctx: *mut vm::Ctx, cb: F) -> R { + let addr = CURRENT_CTX.with(|x| x.get()); + let old = *addr; + *addr = ctx; + let ret = cb(); + *addr = old; + ret +} + +/// Pushes a new `CodeVersion` to the current code versions. +pub fn push_code_version(version: CodeVersion) { + CURRENT_CODE_VERSIONS.with(|x| x.borrow_mut().push(version)); +} + +/// Pops a `CodeVersion` from the current code versions. +pub fn pop_code_version() -> Option { + CURRENT_CODE_VERSIONS.with(|x| x.borrow_mut().pop()) +} + +/// Gets the wasm interrupt signal mem. +pub unsafe fn get_wasm_interrupt_signal_mem() -> *mut u8 { + INTERRUPT_SIGNAL_MEM.0 +} + +/// Sets the wasm interrupt on the given `Ctx`. +pub unsafe fn set_wasm_interrupt_on_ctx(ctx: *mut vm::Ctx) { + if mprotect( + (&*ctx).internal.interrupt_signal_mem as _, + INTERRUPT_SIGNAL_MEM_SIZE, + PROT_NONE, + ) < 0 + { + panic!("cannot set PROT_NONE on signal mem"); + } +} + +/// Sets a wasm interrupt. +pub unsafe fn set_wasm_interrupt() { + let mem: *mut u8 = INTERRUPT_SIGNAL_MEM.0; + if mprotect(mem as _, INTERRUPT_SIGNAL_MEM_SIZE, PROT_NONE) < 0 { + panic!("cannot set PROT_NONE on signal mem"); + } +} + +/// Clears the wasm interrupt. +pub unsafe fn clear_wasm_interrupt() { + let mem: *mut u8 = INTERRUPT_SIGNAL_MEM.0; + if mprotect(mem as _, INTERRUPT_SIGNAL_MEM_SIZE, PROT_READ | PROT_WRITE) < 0 { + panic!("cannot set PROT_READ | PROT_WRITE on signal mem"); + } +} + +/// Catches an unsafe unwind with the given functions and breakpoints. +pub unsafe fn catch_unsafe_unwind R>( + f: F, + breakpoints: Option, +) -> Result> { + let unwind = UNWIND.with(|x| x.get()); + let old = (*unwind).take(); + *unwind = Some(UnwindInfo { + jmpbuf: [0; SETJMP_BUFFER_LEN], + breakpoints: breakpoints, + payload: None, + }); + + if raw::setjmp(&mut (*unwind).as_mut().unwrap().jmpbuf as *mut SetJmpBuffer as *mut _) != 0 { + // error + let ret = (*unwind).as_mut().unwrap().payload.take().unwrap(); + *unwind = old; + Err(ret) + } else { + let ret = f(); + // implicit control flow to the error case... + *unwind = old; + Ok(ret) + } +} + +/// Begins an unsafe unwind. +pub unsafe fn begin_unsafe_unwind(e: Box) -> ! { + let unwind = UNWIND.with(|x| x.get()); + let inner = (*unwind) + .as_mut() + .expect("not within a catch_unsafe_unwind scope"); + inner.payload = Some(e); + raw::longjmp(&mut inner.jmpbuf as *mut SetJmpBuffer as *mut _, 0xffff); +} + +unsafe fn with_breakpoint_map) -> R>(f: F) -> R { + let unwind = UNWIND.with(|x| x.get()); + let inner = (*unwind) + .as_mut() + .expect("not within a catch_unsafe_unwind scope"); + f(inner.breakpoints.as_ref()) +} + +#[cfg(not(target_arch = "x86_64"))] +/// Allocates and runs with the given stack size and closure. +pub fn allocate_and_run R>(_size: usize, f: F) -> R { + f() +} + +#[cfg(target_arch = "x86_64")] +/// Allocates and runs with the given stack size and closure. +pub fn allocate_and_run R>(size: usize, f: F) -> R { + struct Context R, R> { + f: Option, + ret: Option, + } + + extern "C" fn invoke R, R>(ctx: &mut Context) { + let f = ctx.f.take().unwrap(); + ctx.ret = Some(f()); + } + + unsafe { + let mut ctx = Context { + f: Some(f), + ret: None, + }; + assert!(size % 16 == 0); + assert!(size >= 4096); + + let mut stack: Vec = vec![0; size / 8]; + let end_offset = stack.len(); + + stack[end_offset - 4] = invoke:: as usize as u64; + + // NOTE: Keep this consistent with `image-loading-*.s`. + stack[end_offset - 4 - 10] = &mut ctx as *mut Context as usize as u64; // rdi + const NUM_SAVED_REGISTERS: usize = 31; + let stack_begin = stack.as_mut_ptr().add(end_offset - 4 - NUM_SAVED_REGISTERS); + let stack_end = stack.as_mut_ptr().add(end_offset); + + raw::run_on_alternative_stack(stack_end, stack_begin); + ctx.ret.take().unwrap() + } +} + +extern "C" fn signal_trap_handler( + signum: ::nix::libc::c_int, + siginfo: *mut siginfo_t, + ucontext: *mut c_void, +) { + use crate::backend::{Architecture, InlineBreakpointType}; + + #[cfg(target_arch = "x86_64")] + static ARCH: Architecture = Architecture::X64; + + #[cfg(target_arch = "aarch64")] + static ARCH: Architecture = Architecture::Aarch64; + + let mut should_unwind = false; + let mut unwind_result: Box = Box::new(()); + + unsafe { + let fault = get_fault_info(siginfo as _, ucontext); + let early_return = allocate_and_run(TRAP_STACK_SIZE, || { + CURRENT_CODE_VERSIONS.with(|versions| { + let versions = versions.borrow(); + for v in versions.iter() { + let magic_size = + if let Some(x) = v.runnable_module.get_inline_breakpoint_size(ARCH) { + x + } else { + continue; + }; + let ip = fault.ip.get(); + let end = v.base + v.msm.total_size; + if ip >= v.base && ip < end && ip + magic_size <= end { + if let Some(ib) = v.runnable_module.read_inline_breakpoint( + ARCH, + std::slice::from_raw_parts(ip as *const u8, magic_size), + ) { + match ib.ty { + InlineBreakpointType::Middleware => { + let out: Option>> = + with_breakpoint_map(|bkpt_map| { + bkpt_map.and_then(|x| x.get(&ip)).map(|x| { + x(BreakpointInfo { + fault: Some(&fault), + }) + }) + }); + if let Some(Ok(())) = out { + } else if let Some(Err(e)) = out { + should_unwind = true; + unwind_result = e; + } + } + } + + fault.ip.set(ip + magic_size); + return true; + } + break; + } + } + false + }) + }); + if should_unwind { + begin_unsafe_unwind(unwind_result); + } + if early_return { + return; + } + + should_unwind = allocate_and_run(TRAP_STACK_SIZE, || { + let mut is_suspend_signal = false; + + WAS_SIGINT_TRIGGERED.with(|x| x.set(false)); + + match Signal::from_c_int(signum) { + Ok(SIGTRAP) => { + // breakpoint + let out: Option>> = + with_breakpoint_map(|bkpt_map| { + bkpt_map.and_then(|x| x.get(&(fault.ip.get()))).map(|x| { + x(BreakpointInfo { + fault: Some(&fault), + }) + }) + }); + match out { + Some(Ok(())) => { + return false; + } + Some(Err(e)) => { + unwind_result = e; + return true; + } + None => {} + } + } + Ok(SIGSEGV) | Ok(SIGBUS) => { + if fault.faulting_addr as usize == get_wasm_interrupt_signal_mem() as usize { + is_suspend_signal = true; + clear_wasm_interrupt(); + if INTERRUPT_SIGNAL_DELIVERED.swap(false, Ordering::SeqCst) { + WAS_SIGINT_TRIGGERED.with(|x| x.set(true)); + } + } + } + _ => {} + } + + // Now we have looked up all possible handler tables but failed to find a handler + // for this exception that allows a normal return. + // + // So here we check whether this exception is caused by a suspend signal, return the + // state image if so, or throw the exception out otherwise. + + let ctx: &mut vm::Ctx = &mut **CURRENT_CTX.with(|x| x.get()); + let es_image = fault + .read_stack(None) + .expect("fault.read_stack() failed. Broken invariants?"); + + if is_suspend_signal { + // If this is a suspend signal, we parse the runtime state and return the resulting image. + let image = build_instance_image(ctx, es_image); + unwind_result = Box::new(image); + } else { + // Otherwise, this is a real exception and we just throw it to the caller. + if !es_image.frames.is_empty() { + eprintln!( + "\n{}", + "Wasmer encountered an error while running your WebAssembly program." + ); + es_image.print_backtrace_if_needed(); + } + + // Look up the exception tables and try to find an exception code. + let exc_code = CURRENT_CODE_VERSIONS.with(|versions| { + let versions = versions.borrow(); + for v in versions.iter() { + if let Some(table) = v.runnable_module.get_exception_table() { + let ip = fault.ip.get(); + let end = v.base + v.msm.total_size; + if ip >= v.base && ip < end { + if let Some(exc_code) = table.offset_to_code.get(&(ip - v.base)) { + return Some(*exc_code); + } + } + } + } + None + }); + if let Some(code) = exc_code { + unwind_result = Box::new(code); + } + } + + true + }); + + if should_unwind { + begin_unsafe_unwind(unwind_result); + } + } +} + +extern "C" fn sigint_handler( + _signum: ::nix::libc::c_int, + _siginfo: *mut siginfo_t, + _ucontext: *mut c_void, +) { + if INTERRUPT_SIGNAL_DELIVERED.swap(true, Ordering::SeqCst) { + eprintln!("Got another SIGINT before trap is triggered on WebAssembly side, aborting"); + process::abort(); + } + unsafe { + set_wasm_interrupt(); + } +} + +/// Ensure the signal handler is installed. +pub fn ensure_sighandler() { + INSTALL_SIGHANDLER.call_once(|| unsafe { + install_sighandler_as_dylib(); + }); +} + +static INSTALL_SIGHANDLER: Once = Once::new(); + +#[allow(dead_code)] +unsafe fn install_sighandler() { + let sa_trap = SigAction::new( + SigHandler::SigAction(signal_trap_handler), + SaFlags::SA_ONSTACK, + SigSet::empty(), + ); + sigaction(SIGFPE, &sa_trap).unwrap(); + sigaction(SIGILL, &sa_trap).unwrap(); + sigaction(SIGSEGV, &sa_trap).unwrap(); + sigaction(SIGBUS, &sa_trap).unwrap(); + sigaction(SIGTRAP, &sa_trap).unwrap(); + + let sa_interrupt = SigAction::new( + SigHandler::SigAction(sigint_handler), + SaFlags::SA_ONSTACK, + SigSet::empty(), + ); + sigaction(SIGINT, &sa_interrupt).unwrap(); +} + +unsafe fn install_sighandler_as_dylib() { + let sa_trap = SigAction::new( + SigHandler::SigAction(signal_trap_handler), + SaFlags::SA_ONSTACK, + SigSet::empty(), + ); + sigaction(SIGFPE, &sa_trap).unwrap(); + sigaction(SIGILL, &sa_trap).unwrap(); + sigaction(SIGBUS, &sa_trap).unwrap(); + sigaction(SIGTRAP, &sa_trap).unwrap(); + + let capture_sigsegv = !SIGSEGV_PASSTHROUGH.load(Ordering::SeqCst); + if capture_sigsegv { + sigaction(SIGSEGV, &sa_trap).unwrap(); + } +} + +#[derive(Debug, Clone)] +/// Info about the fault +pub struct FaultInfo { + /// Faulting address. + pub faulting_addr: *const c_void, + /// Instruction pointer. + pub ip: &'static Cell, + /// Values of known registers. + pub known_registers: [Option; 32], +} + +impl FaultInfo { + /// Parses the stack and builds an execution state image. + pub unsafe fn read_stack(&self, max_depth: Option) -> Option { + let rsp = self.known_registers[X64Register::GPR(GPR::RSP).to_index().0]?; + + Some(CURRENT_CODE_VERSIONS.with(|versions| { + let versions = versions.borrow(); + read_stack( + || versions.iter(), + rsp as usize as *const u64, + self.known_registers, + Some(self.ip.get() as u64), + max_depth, + ) + })) + } +} + +#[cfg(all(target_os = "freebsd", target_arch = "aarch64"))] +/// Get fault info from siginfo and ucontext. +pub unsafe fn get_fault_info(siginfo: *const c_void, ucontext: *mut c_void) -> FaultInfo { + #[repr(C)] + pub struct ucontext_t { + uc_sigmask: libc::sigset_t, + uc_mcontext: mcontext_t, + uc_link: *mut ucontext_t, + uc_stack: libc::stack_t, + uc_flags: i32, + __spare__: [i32; 4], + } + #[repr(C)] + pub struct gpregs { + gp_x: [u64; 30], + gp_lr: u64, + gp_sp: u64, + gp_elr: u64, + gp_spsr: u64, + gp_pad: i32, + }; + #[repr(C)] + pub struct fpregs { + fp_q: [u128; 32], + fp_sr: u32, + fp_cr: u32, + fp_flags: i32, + fp_pad: i32, + }; + #[repr(C)] + pub struct mcontext_t { + mc_gpregs: gpregs, + mc_fpregs: fpregs, + mc_flags: i32, + mc_pad: i32, + mc_spare: [u64; 8], + } + + let siginfo = siginfo as *const siginfo_t; + let si_addr = (*siginfo).si_addr; + + let ucontext = ucontext as *mut ucontext_t; + let gregs = &(*ucontext).uc_mcontext.mc_gpregs; + + let mut known_registers: [Option; 32] = [None; 32]; + + known_registers[X64Register::GPR(GPR::R15).to_index().0] = Some(gregs.gp_x[15] as _); + known_registers[X64Register::GPR(GPR::R14).to_index().0] = Some(gregs.gp_x[14] as _); + known_registers[X64Register::GPR(GPR::R13).to_index().0] = Some(gregs.gp_x[13] as _); + known_registers[X64Register::GPR(GPR::R12).to_index().0] = Some(gregs.gp_x[12] as _); + known_registers[X64Register::GPR(GPR::R11).to_index().0] = Some(gregs.gp_x[11] as _); + known_registers[X64Register::GPR(GPR::R10).to_index().0] = Some(gregs.gp_x[10] as _); + known_registers[X64Register::GPR(GPR::R9).to_index().0] = Some(gregs.gp_x[9] as _); + known_registers[X64Register::GPR(GPR::R8).to_index().0] = Some(gregs.gp_x[8] as _); + known_registers[X64Register::GPR(GPR::RSI).to_index().0] = Some(gregs.gp_x[6] as _); + known_registers[X64Register::GPR(GPR::RDI).to_index().0] = Some(gregs.gp_x[7] as _); + known_registers[X64Register::GPR(GPR::RDX).to_index().0] = Some(gregs.gp_x[2] as _); + known_registers[X64Register::GPR(GPR::RCX).to_index().0] = Some(gregs.gp_x[1] as _); + known_registers[X64Register::GPR(GPR::RBX).to_index().0] = Some(gregs.gp_x[3] as _); + known_registers[X64Register::GPR(GPR::RAX).to_index().0] = Some(gregs.gp_x[0] as _); + + known_registers[X64Register::GPR(GPR::RBP).to_index().0] = Some(gregs.gp_x[5] as _); + known_registers[X64Register::GPR(GPR::RSP).to_index().0] = Some(gregs.gp_x[28] as _); + + FaultInfo { + faulting_addr: si_addr as usize as _, + ip: std::mem::transmute::<&mut u64, &'static Cell>( + &mut (*ucontext).uc_mcontext.mc_gpregs.gp_elr, + ), + known_registers, + } +} + +#[cfg(all(target_os = "freebsd", target_arch = "x86_64"))] +/// Get fault info from siginfo and ucontext. +pub unsafe fn get_fault_info(siginfo: *const c_void, ucontext: *mut c_void) -> FaultInfo { + use crate::state::x64::XMM; + #[repr(C)] + pub struct ucontext_t { + uc_sigmask: libc::sigset_t, + uc_mcontext: mcontext_t, + uc_link: *mut ucontext_t, + uc_stack: libc::stack_t, + uc_flags: i32, + __spare__: [i32; 4], + } + #[repr(C)] + pub struct mcontext_t { + mc_onstack: u64, + mc_rdi: u64, + mc_rsi: u64, + mc_rdx: u64, + mc_rcx: u64, + mc_r8: u64, + mc_r9: u64, + mc_rax: u64, + mc_rbx: u64, + mc_rbp: u64, + mc_r10: u64, + mc_r11: u64, + mc_r12: u64, + mc_r13: u64, + mc_r14: u64, + mc_r15: u64, + mc_trapno: u32, + mc_fs: u16, + mc_gs: u16, + mc_addr: u64, + mc_flags: u32, + mc_es: u16, + mc_ds: u16, + mc_err: u64, + mc_rip: u64, + mc_cs: u64, + mc_rflags: u64, + mc_rsp: u64, + mc_ss: u64, + mc_len: i64, + + mc_fpformat: i64, + mc_ownedfp: i64, + mc_savefpu: *const savefpu, + mc_fpstate: [i64; 63], // mc_fpstate[0] is a pointer to savefpu + + mc_fsbase: u64, + mc_gsbase: u64, + + mc_xfpustate: u64, + mc_xfpustate_len: u64, + + mc_spare: [i64; 4], + } + #[repr(C)] + pub struct xmmacc { + element: [u32; 4], + } + #[repr(C)] + pub struct __envxmm64 { + en_cw: u16, + en_sw: u16, + en_tw: u8, + en_zero: u8, + en_opcode: u16, + en_rip: u64, + en_rdp: u64, + en_mxcsr: u32, + en_mxcsr_mask: u32, + } + #[repr(C)] + pub struct fpacc87 { + fp_bytes: [u8; 10], + } + #[repr(C)] + pub struct sv_fp { + fp_acc: fpacc87, + fp_pad: [u8; 6], + } + #[repr(C, align(16))] + pub struct savefpu { + sv_env: __envxmm64, + sv_fp_t: [sv_fp; 8], + sv_xmm: [xmmacc; 16], + sv_pad: [u8; 96], + } + + let siginfo = siginfo as *const siginfo_t; + let si_addr = (*siginfo).si_addr; + + let ucontext = ucontext as *mut ucontext_t; + let gregs = &mut (*ucontext).uc_mcontext; + + fn read_xmm(reg: &xmmacc) -> u64 { + (reg.element[0] as u64) | ((reg.element[1] as u64) << 32) + } + + let mut known_registers: [Option; 32] = [None; 32]; + known_registers[X64Register::GPR(GPR::R15).to_index().0] = Some(gregs.mc_r15); + known_registers[X64Register::GPR(GPR::R14).to_index().0] = Some(gregs.mc_r14); + known_registers[X64Register::GPR(GPR::R13).to_index().0] = Some(gregs.mc_r13); + known_registers[X64Register::GPR(GPR::R12).to_index().0] = Some(gregs.mc_r12); + known_registers[X64Register::GPR(GPR::R11).to_index().0] = Some(gregs.mc_r11); + known_registers[X64Register::GPR(GPR::R10).to_index().0] = Some(gregs.mc_r10); + known_registers[X64Register::GPR(GPR::R9).to_index().0] = Some(gregs.mc_r9); + known_registers[X64Register::GPR(GPR::R8).to_index().0] = Some(gregs.mc_r8); + known_registers[X64Register::GPR(GPR::RSI).to_index().0] = Some(gregs.mc_rsi); + known_registers[X64Register::GPR(GPR::RDI).to_index().0] = Some(gregs.mc_rdi); + known_registers[X64Register::GPR(GPR::RDX).to_index().0] = Some(gregs.mc_rdx); + known_registers[X64Register::GPR(GPR::RCX).to_index().0] = Some(gregs.mc_rcx); + known_registers[X64Register::GPR(GPR::RBX).to_index().0] = Some(gregs.mc_rbx); + known_registers[X64Register::GPR(GPR::RAX).to_index().0] = Some(gregs.mc_rax); + + known_registers[X64Register::GPR(GPR::RBP).to_index().0] = Some(gregs.mc_rbp); + known_registers[X64Register::GPR(GPR::RSP).to_index().0] = Some(gregs.mc_rsp); + + // https://lists.freebsd.org/pipermail/freebsd-arch/2011-December/012077.html + // https://people.freebsd.org/~kib/misc/defer_sig.c + const _MC_HASFPXSTATE: u32 = 0x4; + if (gregs.mc_flags & _MC_HASFPXSTATE) == 0 { + // XXX mc_fpstate[0] is actually a pointer to a struct savefpu + let fpregs = &*(*ucontext).uc_mcontext.mc_savefpu; + known_registers[X64Register::XMM(XMM::XMM0).to_index().0] = + Some(read_xmm(&fpregs.sv_xmm[0])); + known_registers[X64Register::XMM(XMM::XMM1).to_index().0] = + Some(read_xmm(&fpregs.sv_xmm[1])); + known_registers[X64Register::XMM(XMM::XMM2).to_index().0] = + Some(read_xmm(&fpregs.sv_xmm[2])); + known_registers[X64Register::XMM(XMM::XMM3).to_index().0] = + Some(read_xmm(&fpregs.sv_xmm[3])); + known_registers[X64Register::XMM(XMM::XMM4).to_index().0] = + Some(read_xmm(&fpregs.sv_xmm[4])); + known_registers[X64Register::XMM(XMM::XMM5).to_index().0] = + Some(read_xmm(&fpregs.sv_xmm[5])); + known_registers[X64Register::XMM(XMM::XMM6).to_index().0] = + Some(read_xmm(&fpregs.sv_xmm[6])); + known_registers[X64Register::XMM(XMM::XMM7).to_index().0] = + Some(read_xmm(&fpregs.sv_xmm[7])); + known_registers[X64Register::XMM(XMM::XMM8).to_index().0] = + Some(read_xmm(&fpregs.sv_xmm[8])); + known_registers[X64Register::XMM(XMM::XMM9).to_index().0] = + Some(read_xmm(&fpregs.sv_xmm[9])); + known_registers[X64Register::XMM(XMM::XMM10).to_index().0] = + Some(read_xmm(&fpregs.sv_xmm[10])); + known_registers[X64Register::XMM(XMM::XMM11).to_index().0] = + Some(read_xmm(&fpregs.sv_xmm[11])); + known_registers[X64Register::XMM(XMM::XMM12).to_index().0] = + Some(read_xmm(&fpregs.sv_xmm[12])); + known_registers[X64Register::XMM(XMM::XMM13).to_index().0] = + Some(read_xmm(&fpregs.sv_xmm[13])); + known_registers[X64Register::XMM(XMM::XMM14).to_index().0] = + Some(read_xmm(&fpregs.sv_xmm[14])); + known_registers[X64Register::XMM(XMM::XMM15).to_index().0] = + Some(read_xmm(&fpregs.sv_xmm[15])); + } + + FaultInfo { + faulting_addr: si_addr, + ip: std::mem::transmute::<&mut u64, &'static Cell>( + &mut (*ucontext).uc_mcontext.mc_rip, + ), + known_registers, + } +} + +#[cfg(all(target_os = "linux", target_arch = "aarch64"))] +/// Get fault info from siginfo and ucontext. +pub unsafe fn get_fault_info(siginfo: *const c_void, ucontext: *mut c_void) -> FaultInfo { + #[allow(dead_code)] + #[allow(non_camel_case_types)] + #[repr(packed)] + struct sigcontext { + fault_address: u64, + regs: [u64; 31], + sp: u64, + pc: u64, + pstate: u64, + reserved: [u8; 4096], + } + + #[allow(dead_code)] + #[allow(non_camel_case_types)] + #[repr(packed)] + struct ucontext { + unknown: [u8; 176], + uc_mcontext: sigcontext, + } + + #[allow(dead_code)] + #[allow(non_camel_case_types)] + #[repr(C)] + struct siginfo_t { + si_signo: i32, + si_errno: i32, + si_code: i32, + si_addr: u64, + // ... + } + + let siginfo = siginfo as *const siginfo_t; + let si_addr = (*siginfo).si_addr; + + let ucontext = ucontext as *mut ucontext; + let gregs = &(*ucontext).uc_mcontext.regs; + + let mut known_registers: [Option; 32] = [None; 32]; + + known_registers[X64Register::GPR(GPR::R15).to_index().0] = Some(gregs[15] as _); + known_registers[X64Register::GPR(GPR::R14).to_index().0] = Some(gregs[14] as _); + known_registers[X64Register::GPR(GPR::R13).to_index().0] = Some(gregs[13] as _); + known_registers[X64Register::GPR(GPR::R12).to_index().0] = Some(gregs[12] as _); + known_registers[X64Register::GPR(GPR::R11).to_index().0] = Some(gregs[11] as _); + known_registers[X64Register::GPR(GPR::R10).to_index().0] = Some(gregs[10] as _); + known_registers[X64Register::GPR(GPR::R9).to_index().0] = Some(gregs[9] as _); + known_registers[X64Register::GPR(GPR::R8).to_index().0] = Some(gregs[8] as _); + known_registers[X64Register::GPR(GPR::RSI).to_index().0] = Some(gregs[6] as _); + known_registers[X64Register::GPR(GPR::RDI).to_index().0] = Some(gregs[7] as _); + known_registers[X64Register::GPR(GPR::RDX).to_index().0] = Some(gregs[2] as _); + known_registers[X64Register::GPR(GPR::RCX).to_index().0] = Some(gregs[1] as _); + known_registers[X64Register::GPR(GPR::RBX).to_index().0] = Some(gregs[3] as _); + known_registers[X64Register::GPR(GPR::RAX).to_index().0] = Some(gregs[0] as _); + + known_registers[X64Register::GPR(GPR::RBP).to_index().0] = Some(gregs[5] as _); + known_registers[X64Register::GPR(GPR::RSP).to_index().0] = Some(gregs[28] as _); + + FaultInfo { + faulting_addr: si_addr as usize as _, + ip: std::mem::transmute::<&mut u64, &'static Cell>(&mut (*ucontext).uc_mcontext.pc), + known_registers, + } +} + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +/// Get fault info from siginfo and ucontext. +pub unsafe fn get_fault_info(siginfo: *const c_void, ucontext: *mut c_void) -> FaultInfo { + use libc::{ + ucontext_t, REG_R10, REG_R11, REG_R12, REG_R13, REG_R14, REG_R15, REG_R8, REG_R9, REG_RAX, + REG_RBP, REG_RBX, REG_RCX, REG_RDI, REG_RDX, REG_RIP, REG_RSI, REG_RSP, + }; + + #[cfg(not(target_env = "musl"))] + fn read_xmm(reg: &libc::_libc_xmmreg) -> u64 { + (reg.element[0] as u64) | ((reg.element[1] as u64) << 32) + } + + #[allow(dead_code)] + #[repr(C)] + struct siginfo_t { + si_signo: i32, + si_errno: i32, + si_code: i32, + si_addr: u64, + // ... + } + + let siginfo = siginfo as *const siginfo_t; + let si_addr = (*siginfo).si_addr; + + let ucontext = ucontext as *mut ucontext_t; + let gregs = &mut (*ucontext).uc_mcontext.gregs; + + let mut known_registers: [Option; 32] = [None; 32]; + known_registers[X64Register::GPR(GPR::R15).to_index().0] = Some(gregs[REG_R15 as usize] as _); + known_registers[X64Register::GPR(GPR::R14).to_index().0] = Some(gregs[REG_R14 as usize] as _); + known_registers[X64Register::GPR(GPR::R13).to_index().0] = Some(gregs[REG_R13 as usize] as _); + known_registers[X64Register::GPR(GPR::R12).to_index().0] = Some(gregs[REG_R12 as usize] as _); + known_registers[X64Register::GPR(GPR::R11).to_index().0] = Some(gregs[REG_R11 as usize] as _); + known_registers[X64Register::GPR(GPR::R10).to_index().0] = Some(gregs[REG_R10 as usize] as _); + known_registers[X64Register::GPR(GPR::R9).to_index().0] = Some(gregs[REG_R9 as usize] as _); + known_registers[X64Register::GPR(GPR::R8).to_index().0] = Some(gregs[REG_R8 as usize] as _); + known_registers[X64Register::GPR(GPR::RSI).to_index().0] = Some(gregs[REG_RSI as usize] as _); + known_registers[X64Register::GPR(GPR::RDI).to_index().0] = Some(gregs[REG_RDI as usize] as _); + known_registers[X64Register::GPR(GPR::RDX).to_index().0] = Some(gregs[REG_RDX as usize] as _); + known_registers[X64Register::GPR(GPR::RCX).to_index().0] = Some(gregs[REG_RCX as usize] as _); + known_registers[X64Register::GPR(GPR::RBX).to_index().0] = Some(gregs[REG_RBX as usize] as _); + known_registers[X64Register::GPR(GPR::RAX).to_index().0] = Some(gregs[REG_RAX as usize] as _); + + known_registers[X64Register::GPR(GPR::RBP).to_index().0] = Some(gregs[REG_RBP as usize] as _); + known_registers[X64Register::GPR(GPR::RSP).to_index().0] = Some(gregs[REG_RSP as usize] as _); + + // Skip reading floating point registers when building with musl libc. + // FIXME: Depends on https://github.com/rust-lang/libc/pull/1646 + #[cfg(not(target_env = "musl"))] + { + use crate::state::x64::XMM; + if !(*ucontext).uc_mcontext.fpregs.is_null() { + let fpregs = &*(*ucontext).uc_mcontext.fpregs; + known_registers[X64Register::XMM(XMM::XMM0).to_index().0] = + Some(read_xmm(&fpregs._xmm[0])); + known_registers[X64Register::XMM(XMM::XMM1).to_index().0] = + Some(read_xmm(&fpregs._xmm[1])); + known_registers[X64Register::XMM(XMM::XMM2).to_index().0] = + Some(read_xmm(&fpregs._xmm[2])); + known_registers[X64Register::XMM(XMM::XMM3).to_index().0] = + Some(read_xmm(&fpregs._xmm[3])); + known_registers[X64Register::XMM(XMM::XMM4).to_index().0] = + Some(read_xmm(&fpregs._xmm[4])); + known_registers[X64Register::XMM(XMM::XMM5).to_index().0] = + Some(read_xmm(&fpregs._xmm[5])); + known_registers[X64Register::XMM(XMM::XMM6).to_index().0] = + Some(read_xmm(&fpregs._xmm[6])); + known_registers[X64Register::XMM(XMM::XMM7).to_index().0] = + Some(read_xmm(&fpregs._xmm[7])); + known_registers[X64Register::XMM(XMM::XMM8).to_index().0] = + Some(read_xmm(&fpregs._xmm[8])); + known_registers[X64Register::XMM(XMM::XMM9).to_index().0] = + Some(read_xmm(&fpregs._xmm[9])); + known_registers[X64Register::XMM(XMM::XMM10).to_index().0] = + Some(read_xmm(&fpregs._xmm[10])); + known_registers[X64Register::XMM(XMM::XMM11).to_index().0] = + Some(read_xmm(&fpregs._xmm[11])); + known_registers[X64Register::XMM(XMM::XMM12).to_index().0] = + Some(read_xmm(&fpregs._xmm[12])); + known_registers[X64Register::XMM(XMM::XMM13).to_index().0] = + Some(read_xmm(&fpregs._xmm[13])); + known_registers[X64Register::XMM(XMM::XMM14).to_index().0] = + Some(read_xmm(&fpregs._xmm[14])); + known_registers[X64Register::XMM(XMM::XMM15).to_index().0] = + Some(read_xmm(&fpregs._xmm[15])); + } + } + + FaultInfo { + faulting_addr: si_addr as usize as _, + ip: std::mem::transmute::<&mut i64, &'static Cell>(&mut gregs[REG_RIP as usize]), + known_registers, + } +} + +/// Get fault info from siginfo and ucontext. +#[cfg(all(target_os = "macos", target_arch = "x86_64"))] +pub unsafe fn get_fault_info(siginfo: *const c_void, ucontext: *mut c_void) -> FaultInfo { + use crate::state::x64::XMM; + #[allow(dead_code)] + #[repr(C)] + struct ucontext_t { + uc_onstack: u32, + uc_sigmask: u32, + uc_stack: libc::stack_t, + uc_link: *const ucontext_t, + uc_mcsize: u64, + uc_mcontext: *mut mcontext_t, + } + #[repr(C)] + struct exception_state { + trapno: u16, + cpu: u16, + err: u32, + faultvaddr: u64, + } + #[repr(C)] + struct regs { + rax: u64, + rbx: u64, + rcx: u64, + rdx: u64, + rdi: u64, + rsi: u64, + rbp: u64, + rsp: u64, + r8: u64, + r9: u64, + r10: u64, + r11: u64, + r12: u64, + r13: u64, + r14: u64, + r15: u64, + rip: u64, + rflags: u64, + cs: u64, + fs: u64, + gs: u64, + } + #[repr(C)] + struct fpstate { + _cwd: u16, + _swd: u16, + _ftw: u16, + _fop: u16, + _rip: u64, + _rdp: u64, + _mxcsr: u32, + _mxcr_mask: u32, + _st: [[u16; 8]; 8], + xmm: [[u64; 2]; 16], + _padding: [u32; 24], + } + #[allow(dead_code)] + #[repr(C)] + struct mcontext_t { + es: exception_state, + ss: regs, + fs: fpstate, + } + + let siginfo = siginfo as *const siginfo_t; + let si_addr = (*siginfo).si_addr; + + let ucontext = ucontext as *mut ucontext_t; + let ss = &mut (*(*ucontext).uc_mcontext).ss; + let fs = &(*(*ucontext).uc_mcontext).fs; + + let mut known_registers: [Option; 32] = [None; 32]; + + known_registers[X64Register::GPR(GPR::R15).to_index().0] = Some(ss.r15); + known_registers[X64Register::GPR(GPR::R14).to_index().0] = Some(ss.r14); + known_registers[X64Register::GPR(GPR::R13).to_index().0] = Some(ss.r13); + known_registers[X64Register::GPR(GPR::R12).to_index().0] = Some(ss.r12); + known_registers[X64Register::GPR(GPR::R11).to_index().0] = Some(ss.r11); + known_registers[X64Register::GPR(GPR::R10).to_index().0] = Some(ss.r10); + known_registers[X64Register::GPR(GPR::R9).to_index().0] = Some(ss.r9); + known_registers[X64Register::GPR(GPR::R8).to_index().0] = Some(ss.r8); + known_registers[X64Register::GPR(GPR::RSI).to_index().0] = Some(ss.rsi); + known_registers[X64Register::GPR(GPR::RDI).to_index().0] = Some(ss.rdi); + known_registers[X64Register::GPR(GPR::RDX).to_index().0] = Some(ss.rdx); + known_registers[X64Register::GPR(GPR::RCX).to_index().0] = Some(ss.rcx); + known_registers[X64Register::GPR(GPR::RBX).to_index().0] = Some(ss.rbx); + known_registers[X64Register::GPR(GPR::RAX).to_index().0] = Some(ss.rax); + + known_registers[X64Register::GPR(GPR::RBP).to_index().0] = Some(ss.rbp); + known_registers[X64Register::GPR(GPR::RSP).to_index().0] = Some(ss.rsp); + + known_registers[X64Register::XMM(XMM::XMM0).to_index().0] = Some(fs.xmm[0][0]); + known_registers[X64Register::XMM(XMM::XMM1).to_index().0] = Some(fs.xmm[1][0]); + known_registers[X64Register::XMM(XMM::XMM2).to_index().0] = Some(fs.xmm[2][0]); + known_registers[X64Register::XMM(XMM::XMM3).to_index().0] = Some(fs.xmm[3][0]); + known_registers[X64Register::XMM(XMM::XMM4).to_index().0] = Some(fs.xmm[4][0]); + known_registers[X64Register::XMM(XMM::XMM5).to_index().0] = Some(fs.xmm[5][0]); + known_registers[X64Register::XMM(XMM::XMM6).to_index().0] = Some(fs.xmm[6][0]); + known_registers[X64Register::XMM(XMM::XMM7).to_index().0] = Some(fs.xmm[7][0]); + known_registers[X64Register::XMM(XMM::XMM8).to_index().0] = Some(fs.xmm[8][0]); + known_registers[X64Register::XMM(XMM::XMM9).to_index().0] = Some(fs.xmm[9][0]); + known_registers[X64Register::XMM(XMM::XMM10).to_index().0] = Some(fs.xmm[10][0]); + known_registers[X64Register::XMM(XMM::XMM11).to_index().0] = Some(fs.xmm[11][0]); + known_registers[X64Register::XMM(XMM::XMM12).to_index().0] = Some(fs.xmm[12][0]); + known_registers[X64Register::XMM(XMM::XMM13).to_index().0] = Some(fs.xmm[13][0]); + known_registers[X64Register::XMM(XMM::XMM14).to_index().0] = Some(fs.xmm[14][0]); + known_registers[X64Register::XMM(XMM::XMM15).to_index().0] = Some(fs.xmm[15][0]); + + FaultInfo { + faulting_addr: si_addr, + ip: std::mem::transmute::<&mut u64, &'static Cell>(&mut ss.rip), + known_registers, + } +} diff --git a/lib/runtime-core/src/lib.rs b/lib/runtime-core/src/lib.rs new file mode 100644 index 000000000000..daf63ef74193 --- /dev/null +++ b/lib/runtime-core/src/lib.rs @@ -0,0 +1,192 @@ +//! Wasmer Runtime Core Library +//! +//! This crate provides common data structures which are shared by compiler backends +//! to implement a WebAssembly runtime. +//! +//! This crate also provides an API for users who use wasmer as an embedded wasm runtime which +//! allows operations like compiling, instantiating, providing imports, access exports, memories, +//! and tables for example. +//! +//! Most wasmer users should prefer the API which is re-exported by the `wasmer-runtime` +//! library by default. This crate provides additional APIs which may be useful to users +//! that wish to customize the wasmer runtime. +//! + +#![deny( + dead_code, + missing_docs, + nonstandard_style, + unused_imports, + unused_mut, + unused_variables, + unused_unsafe, + unreachable_patterns +)] +#![cfg_attr(nightly, feature(unwind_attributes))] +#![doc(html_favicon_url = "https://wasmer.io/static/icons/favicon.ico")] +#![doc(html_logo_url = "https://avatars3.githubusercontent.com/u/44205449?s=200&v=4")] + +#[macro_use] +extern crate serde_derive; + +#[allow(unused_imports)] +#[macro_use] +extern crate lazy_static; + +#[macro_use] +mod macros; +#[doc(hidden)] +pub mod backend; +mod backing; + +pub mod cache; +pub mod codegen; +pub mod error; +pub mod export; +pub mod global; +pub mod import; +pub mod instance; +pub mod loader; +pub mod memory; +pub mod module; +pub mod parse; +mod sig_registry; +pub mod structures; +mod sys; +pub mod table; +#[cfg(all(unix, target_arch = "x86_64"))] +pub mod trampoline_x64; +pub mod typed_func; +pub mod types; +pub mod units; +pub mod vm; +#[doc(hidden)] +pub mod vmcalls; +#[cfg(all(unix, target_arch = "x86_64"))] +pub use trampoline_x64 as trampoline; +#[cfg(unix)] +pub mod fault; +#[cfg(feature = "generate-debug-information")] +pub mod jit_debug; +pub mod state; +#[cfg(feature = "managed")] +pub mod tiering; + +use self::error::CompileResult; +#[doc(inline)] +pub use self::error::Result; +#[doc(inline)] +pub use self::import::IsExport; +#[doc(inline)] +pub use self::instance::{DynFunc, Instance}; +#[doc(inline)] +pub use self::module::Module; +#[doc(inline)] +pub use self::typed_func::Func; +use std::sync::Arc; + +pub use wasmparser; + +use self::cache::{Artifact, Error as CacheError}; + +pub mod prelude { + //! The prelude module is a helper module used to bring commonly used runtime core imports into + //! scope. + + pub use crate::import::{ImportObject, Namespace}; + pub use crate::types::{ + FuncIndex, GlobalIndex, ImportedFuncIndex, ImportedGlobalIndex, ImportedMemoryIndex, + ImportedTableIndex, LocalFuncIndex, LocalGlobalIndex, LocalMemoryIndex, LocalTableIndex, + MemoryIndex, TableIndex, Type, Value, + }; + pub use crate::vm; + pub use crate::{func, imports}; +} + +pub use crate::import::{ImportObject, Namespace}; + +/// Compile a [`Module`] using the provided compiler from +/// WebAssembly binary code. This function is useful if it +/// is necessary to a compile a module before it can be instantiated +/// and must be used if you wish to use a different backend from the default. +pub fn compile_with( + wasm: &[u8], + compiler: &dyn backend::Compiler, +) -> CompileResult { + let token = backend::Token::generate(); + compiler + .compile(wasm, Default::default(), token) + .map(|mut inner| { + let inner_info: &mut crate::module::ModuleInfo = &mut inner.info; + inner_info.import_custom_sections(wasm).unwrap(); + module::Module::new(Arc::new(inner)) + }) +} + +/// The same as `compile_with` but changes the compiler behavior +/// with the values in the `CompilerConfig` +pub fn compile_with_config( + wasm: &[u8], + compiler: &dyn backend::Compiler, + compiler_config: backend::CompilerConfig, +) -> CompileResult { + let token = backend::Token::generate(); + compiler + .compile(wasm, compiler_config, token) + .map(|inner| module::Module::new(Arc::new(inner))) +} + +/// Perform validation as defined by the +/// WebAssembly specification. Returns `true` if validation +/// succeeded, `false` if validation failed. +pub fn validate(wasm: &[u8]) -> bool { + validate_and_report_errors(wasm).is_ok() +} + +/// The same as `validate` but with an Error message on failure +pub fn validate_and_report_errors(wasm: &[u8]) -> ::std::result::Result<(), String> { + validate_and_report_errors_with_features(wasm, Default::default()) +} + +/// The same as `validate_and_report_errors` but with a Features. +pub fn validate_and_report_errors_with_features( + wasm: &[u8], + features: backend::Features, +) -> ::std::result::Result<(), String> { + use wasmparser::WasmDecoder; + let config = wasmparser::ValidatingParserConfig { + operator_config: wasmparser::OperatorValidatorConfig { + enable_simd: features.simd, + enable_bulk_memory: false, + enable_multi_value: false, + enable_reference_types: false, + enable_threads: features.threads, + + #[cfg(feature = "deterministic-execution")] + deterministic_only: true, + }, + }; + let mut parser = wasmparser::ValidatingParser::new(wasm, Some(config)); + loop { + let state = parser.read(); + match *state { + wasmparser::ParserState::EndWasm => break Ok(()), + wasmparser::ParserState::Error(ref e) => break Err(format!("{}", e)), + _ => {} + } + } +} + +/// Creates a new module from the given cache [`Artifact`] for the specified compiler backend +pub unsafe fn load_cache_with( + cache: Artifact, + compiler: &dyn backend::Compiler, +) -> std::result::Result { + let token = backend::Token::generate(); + compiler + .from_cache(cache, token) + .map(|inner| module::Module::new(Arc::new(inner))) +} + +/// The current version of this crate +pub const VERSION: &str = env!("CARGO_PKG_VERSION"); diff --git a/lib/runtime-core/src/memory/mod.rs b/lib/runtime-core/src/memory/mod.rs new file mode 100644 index 000000000000..83797d65266b --- /dev/null +++ b/lib/runtime-core/src/memory/mod.rs @@ -0,0 +1,383 @@ +//! The memory module contains the implementation data structures and helper functions used to +//! manipulate and access wasm memory. +use crate::{ + error::{CreationError, GrowError}, + export::Export, + import::IsExport, + memory::dynamic::DYNAMIC_GUARD_SIZE, + memory::static_::{SAFE_STATIC_GUARD_SIZE, SAFE_STATIC_HEAP_SIZE}, + types::{MemoryDescriptor, ValueType}, + units::Pages, + vm, +}; +use std::{cell::Cell, fmt, mem, sync::Arc}; + +use std::sync::Mutex as StdMutex; + +use rkyv::{Archive, Serialize as RkyvSerialize, Deserialize as RkyvDeserialize}; + +pub use self::dynamic::DynamicMemory; +pub use self::static_::StaticMemory; +pub use self::view::{Atomically, MemoryView}; + +use parking_lot::Mutex; + +mod dynamic; +pub mod ptr; +mod static_; +mod view; + +#[derive(Clone)] +enum MemoryVariant { + Unshared(UnsharedMemory), + Shared(SharedMemory), +} + +/// A shared or unshared wasm linear memory. +/// +/// A `Memory` represents the memory used by a wasm instance. +#[derive(Clone)] +pub struct Memory { + desc: MemoryDescriptor, + variant: MemoryVariant, +} + +impl Memory { + /// Create a new `Memory` from a [`MemoryDescriptor`] + /// + /// [`MemoryDescriptor`]: struct.MemoryDescriptor.html + /// + /// Usage: + /// + /// ``` + /// # use wasmer_runtime_core::types::MemoryDescriptor; + /// # use wasmer_runtime_core::memory::Memory; + /// # use wasmer_runtime_core::error::Result; + /// # use wasmer_runtime_core::units::Pages; + /// fn create_memory() -> Result<()> { + /// let descriptor = MemoryDescriptor::new(Pages(10), None, false).unwrap(); + /// + /// let memory = Memory::new(descriptor)?; + /// Ok(()) + /// } + /// ``` + pub fn new(desc: MemoryDescriptor) -> Result { + if let Some(max) = desc.maximum { + if max < desc.minimum { + return Err(CreationError::InvalidDescriptor( + "Max number of memory pages is less than the minimum number of pages" + .to_string(), + )); + } + } + + if desc.shared && desc.maximum.is_none() { + return Err(CreationError::InvalidDescriptor( + "Max number of pages is required for shared memory".to_string(), + )); + } + + let variant = if !desc.shared { + MemoryVariant::Unshared(UnsharedMemory::new(desc)?) + } else { + MemoryVariant::Shared(SharedMemory::new(desc)?) + }; + + Ok(Memory { desc, variant }) + } + + /// Return the [`MemoryDescriptor`] that this memory + /// was created with. + /// + /// [`MemoryDescriptor`]: struct.MemoryDescriptor.html + pub fn descriptor(&self) -> MemoryDescriptor { + self.desc + } + + /// Grow this memory by the specified number of pages. + pub fn grow(&self, delta: Pages) -> Result { + match &self.variant { + MemoryVariant::Unshared(unshared_mem) => unshared_mem.grow(delta), + MemoryVariant::Shared(shared_mem) => shared_mem.grow(delta), + } + } + + /// The size, in wasm pages, of this memory. + pub fn size(&self) -> Pages { + match &self.variant { + MemoryVariant::Unshared(unshared_mem) => unshared_mem.size(), + MemoryVariant::Shared(shared_mem) => shared_mem.size(), + } + } + + /// Return a "view" of the currently accessible memory. By + /// default, the view is unsynchronized, using regular memory + /// accesses. You can force a memory view to use atomic accesses + /// by calling the [`atomically`] method. + /// + /// [`atomically`]: memory/struct.MemoryView.html#method.atomically + /// + /// # Notes: + /// + /// This method is safe (as in, it won't cause the host to crash or have UB), + /// but it doesn't obey rust's rules involving data races, especially concurrent ones. + /// Therefore, if this memory is shared between multiple threads, a single memory + /// location can be mutated concurrently without synchronization. + /// + /// # Usage: + /// + /// ``` + /// # use wasmer_runtime_core::memory::{Memory, MemoryView}; + /// # use std::{cell::Cell, sync::atomic::Ordering}; + /// # fn view_memory(memory: Memory) { + /// // Without synchronization. + /// let view: MemoryView = memory.view(); + /// for byte in view[0x1000 .. 0x1010].iter().map(Cell::get) { + /// println!("byte: {}", byte); + /// } + /// + /// // With synchronization. + /// let atomic_view = view.atomically(); + /// for byte in atomic_view[0x1000 .. 0x1010].iter().map(|atom| atom.load(Ordering::SeqCst)) { + /// println!("byte: {}", byte); + /// } + /// # } + /// ``` + pub fn view(&self) -> MemoryView { + let vm::LocalMemory { base, .. } = unsafe { *self.vm_local_memory() }; + + let length = self.size().bytes().0 / mem::size_of::(); + + unsafe { MemoryView::new(base as _, length as u32) } + } + + pub(crate) fn vm_local_memory(&self) -> *mut vm::LocalMemory { + match &self.variant { + MemoryVariant::Unshared(unshared_mem) => unshared_mem.vm_local_memory(), + MemoryVariant::Shared(shared_mem) => shared_mem.vm_local_memory(), + } + } +} + +impl IsExport for Memory { + fn to_export(&self) -> Export { + Export::Memory(self.clone()) + } +} + +impl fmt::Debug for Memory { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("Memory") + .field("desc", &self.desc) + .field("size", &self.size()) + .finish() + } +} + +/// A kind a memory. +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Hash, Archive, RkyvSerialize, RkyvDeserialize)] +pub enum MemoryType { + /// A dynamic memory. + Dynamic, + /// A static memory. + Static, + /// A shared static memory. + SharedStatic, +} + +impl MemoryType { + #[doc(hidden)] + pub fn guard_size(self) -> u64 { + match self { + MemoryType::Dynamic => DYNAMIC_GUARD_SIZE as u64, + MemoryType::Static | MemoryType::SharedStatic => SAFE_STATIC_GUARD_SIZE as u64, + } + } + + #[doc(hidden)] + pub fn bounds(self) -> Option { + match self { + MemoryType::Dynamic => None, + MemoryType::Static | MemoryType::SharedStatic => Some(SAFE_STATIC_HEAP_SIZE as u64), + } + } +} + +enum UnsharedMemoryStorage { + Dynamic(Box), + Static(Box), +} + +/// A reference to an unshared memory. +pub struct UnsharedMemory { + internal: Arc, +} + +struct UnsharedMemoryInternal { + storage: StdMutex, + local: Cell, +} + +// Manually implemented because UnsharedMemoryInternal uses `Cell` and is used in an Arc; +// this is safe because the lock for storage can be used to protect (seems like a weak reason: PLEASE REVIEW!) +unsafe impl Sync for UnsharedMemoryInternal {} + +impl UnsharedMemory { + /// Create a new `UnsharedMemory` from the given memory descriptor. + pub fn new(desc: MemoryDescriptor) -> Result { + let mut local = vm::LocalMemory { + base: std::ptr::null_mut(), + bound: 0, + memory: std::ptr::null_mut(), + }; + + let storage = match desc.memory_type() { + MemoryType::Dynamic => { + UnsharedMemoryStorage::Dynamic(DynamicMemory::new(desc, &mut local)?) + } + MemoryType::Static => { + UnsharedMemoryStorage::Static(StaticMemory::new(desc, &mut local)?) + } + MemoryType::SharedStatic => { + return Err(CreationError::InvalidDescriptor( + "attempting to create shared unshared memory".to_string(), + )); + } + }; + + Ok(Self { + internal: Arc::new(UnsharedMemoryInternal { + storage: StdMutex::new(storage), + local: Cell::new(local), + }), + }) + } + + /// Try to grow this memory by the given number of delta pages. + pub fn grow(&self, delta: Pages) -> Result { + let mut storage = self.internal.storage.lock().unwrap(); + + let mut local = self.internal.local.get(); + + let pages = match &mut *storage { + UnsharedMemoryStorage::Dynamic(dynamic_memory) => { + dynamic_memory.grow(delta, &mut local) + } + UnsharedMemoryStorage::Static(static_memory) => static_memory.grow(delta, &mut local), + }; + + self.internal.local.set(local); + + pages + } + + /// Size of this memory in pages. + pub fn size(&self) -> Pages { + let storage = self.internal.storage.lock().unwrap(); + + match &*storage { + UnsharedMemoryStorage::Dynamic(ref dynamic_memory) => dynamic_memory.size(), + UnsharedMemoryStorage::Static(ref static_memory) => static_memory.size(), + } + } + + pub(crate) fn vm_local_memory(&self) -> *mut vm::LocalMemory { + self.internal.local.as_ptr() + } +} + +impl Clone for UnsharedMemory { + fn clone(&self) -> Self { + UnsharedMemory { + internal: Arc::clone(&self.internal), + } + } +} + +/// A reference to a shared memory. +pub struct SharedMemory { + internal: Arc, +} + +/// Data structure for a shared internal memory. +pub struct SharedMemoryInternal { + memory: StdMutex>, + local: Cell, + lock: Mutex<()>, +} + +// Manually implemented because SharedMemoryInternal uses `Cell` and is used in Arc; +// this is safe because of `lock`; accesing `local` without locking `lock` is not safe (Maybe we could put the lock on Local then?) +unsafe impl Sync for SharedMemoryInternal {} + +impl SharedMemory { + fn new(desc: MemoryDescriptor) -> Result { + let mut local = vm::LocalMemory { + base: std::ptr::null_mut(), + bound: 0, + memory: std::ptr::null_mut(), + }; + + let memory = StaticMemory::new(desc, &mut local)?; + + Ok(Self { + internal: Arc::new(SharedMemoryInternal { + memory: StdMutex::new(memory), + local: Cell::new(local), + lock: Mutex::new(()), + }), + }) + } + + /// Try to grow this memory by the given number of delta pages. + pub fn grow(&self, delta: Pages) -> Result { + let _guard = self.internal.lock.lock(); + let mut local = self.internal.local.get(); + let mut memory = self.internal.memory.lock().unwrap(); + let pages = memory.grow(delta, &mut local); + pages + } + + /// Size of this memory in pages. + pub fn size(&self) -> Pages { + let _guard = self.internal.lock.lock(); + let memory = self.internal.memory.lock().unwrap(); + memory.size() + } + + /// Gets a mutable pointer to the `LocalMemory`. + // This function is scary, because the mutex is not locked here + pub(crate) fn vm_local_memory(&self) -> *mut vm::LocalMemory { + self.internal.local.as_ptr() + } +} + +impl Clone for SharedMemory { + fn clone(&self) -> Self { + SharedMemory { + internal: Arc::clone(&self.internal), + } + } +} + +#[cfg(test)] +mod memory_tests { + + use super::{Memory, MemoryDescriptor, Pages}; + + #[test] + fn test_initial_memory_size() { + let memory_desc = MemoryDescriptor::new(Pages(10), Some(Pages(20)), false).unwrap(); + let unshared_memory = Memory::new(memory_desc).unwrap(); + assert_eq!(unshared_memory.size(), Pages(10)); + } + + #[test] + fn test_invalid_descriptor_returns_error() { + let memory_desc = MemoryDescriptor::new(Pages(10), None, true); + assert!( + memory_desc.is_err(), + "Max number of pages is required for shared memory" + ) + } +} diff --git a/lib/runtime-core/src/module.rs b/lib/runtime-core/src/module.rs new file mode 100644 index 000000000000..41c280d99477 --- /dev/null +++ b/lib/runtime-core/src/module.rs @@ -0,0 +1,354 @@ +//! The module module contains the implementation data structures and helper functions used to +//! manipulate and access wasm modules. +use crate::{ + backend::RunnableModule, + cache::{Artifact, Error as CacheError}, + error, + import::ImportObject, + structures::{Map, TypedIndex}, + types::{ + FuncIndex, FuncSig, GlobalDescriptor, GlobalIndex, GlobalInit, ImportedFuncIndex, + ImportedGlobalIndex, ImportedMemoryIndex, ImportedTableIndex, Initializer, + LocalGlobalIndex, LocalMemoryIndex, LocalTableIndex, MemoryDescriptor, MemoryIndex, + SigIndex, TableDescriptor, TableIndex, + }, + Instance, +}; + +use indexmap::IndexMap; + +use rkyv::{Archive, Serialize as RkyvSerialize, Deserialize as RkyvDeserialize}; + +use crate::backend::CacheGen; +#[cfg(feature = "generate-debug-information")] +use crate::jit_debug; +use std::collections::HashMap; +use std::sync::Arc; + +/// This is used to instantiate a new WebAssembly module. +#[doc(hidden)] +pub struct ModuleInner { + pub runnable_module: Arc>, + pub cache_gen: Box, + pub info: ModuleInfo, +} + +/// Container for module data including memories, globals, tables, imports, and exports. +#[derive(Clone, Debug, Serialize, Deserialize, Archive, RkyvSerialize, RkyvDeserialize)] +pub struct ModuleInfo { + /// Map of memory index to memory descriptors. + // This are strictly local and the typesystem ensures that. + pub memories: Map, + /// Map of global index to global descriptors. + pub globals: Map, + /// Map of table index to table descriptors. + pub tables: Map, + + /// Map of imported function index to import name. + // These are strictly imported and the typesystem ensures that. + pub imported_functions: Map, + /// Map of imported memory index to import name and memory descriptor. + pub imported_memories: Map, + /// Map of imported table index to import name and table descriptor. + pub imported_tables: Map, + /// Map of imported global index to import name and global descriptor. + pub imported_globals: Map, + + /// Map of string to export index. + pub exports: IndexMap, + + /// Vector of data initializers. + pub data_initializers: Vec, + /// Vector of table initializers. + pub elem_initializers: Vec, + + /// Index of optional start function. + pub start_func: Option, + + /// Map function index to signature index. + pub func_assoc: Map, + /// Map signature index to function signature. + pub signatures: Map, + /// Backend. + pub backend: String, + + /// Table of namespace indexes. + pub namespace_table: StringTable, + /// Table of name indexes. + pub name_table: StringTable, + + /// Symbol information from emscripten. + pub em_symbol_map: Option>, + + /// Custom sections. + pub custom_sections: HashMap>, + + /// Flag controlling whether or not debug information for use in a debugger + /// will be generated. + pub generate_debug_info: bool, + + #[cfg(feature = "generate-debug-information")] + #[serde(skip)] + /// Resource manager of debug information being used by a debugger. + pub(crate) debug_info_manager: jit_debug::JitCodeDebugInfoManager, +} + +impl ModuleInfo { + /// Creates custom section info from the given wasm file. + pub fn import_custom_sections(&mut self, wasm: &[u8]) -> crate::error::ParseResult<()> { + let mut parser = wasmparser::ModuleReader::new(wasm)?; + while !parser.eof() { + let section = parser.read()?; + if let wasmparser::SectionCode::Custom { name, kind: _ } = section.code { + let mut reader = section.get_binary_reader(); + let len = reader.bytes_remaining(); + let bytes = reader.read_bytes(len)?; + let data = bytes.to_vec(); + let name = name.to_string(); + self.custom_sections.insert(name, data); + } + } + Ok(()) + } +} + +/// A compiled WebAssembly module. +/// +/// `Module` is returned by the [`compile_with`][] function. +/// +/// [`compile_with`]: crate::compile_with +pub struct Module { + inner: Arc, +} + +impl Module { + pub(crate) fn new(inner: Arc) -> Self { + Module { inner } + } + + /// Instantiate a WebAssembly module with the provided [`ImportObject`]. + /// + /// [`ImportObject`]: struct.ImportObject.html + /// + /// # Note: + /// Instantiating a `Module` will also call the function designated as `start` + /// in the WebAssembly module, if there is one. + /// + /// # Usage: + /// ``` + /// # use wasmer_runtime_core::error::Result; + /// # use wasmer_runtime_core::Module; + /// # use wasmer_runtime_core::imports; + /// # fn instantiate(module: &Module) -> Result<()> { + /// let import_object = imports! { + /// // ... + /// }; + /// let instance = module.instantiate(&import_object)?; + /// // ... + /// # Ok(()) + /// # } + /// ``` + pub fn instantiate(&self, import_object: &ImportObject) -> error::Result { + Instance::new(Arc::clone(&self.inner), import_object) + } + + /// Create a cache artifact from this module. + pub fn cache(&self) -> Result { + let (backend_metadata, code) = self.inner.cache_gen.generate_cache()?; + Ok(Artifact::from_parts( + Box::new(self.inner.info.clone()), + backend_metadata, + code, + )) + } + + /// Get the module data for this module. + pub fn info(&self) -> &ModuleInfo { + &self.inner.info + } +} + +impl Clone for Module { + fn clone(&self) -> Self { + Self { + inner: Arc::clone(&self.inner), + } + } +} + +impl ModuleInner {} + +#[doc(hidden)] +#[derive(Serialize, Deserialize, Debug, Clone, Archive, RkyvSerialize, RkyvDeserialize)] +pub struct ImportName { + pub namespace_index: NamespaceIndex, + pub name_index: NameIndex, +} + +/// A wrapper around the [`TypedIndex`]es for Wasm functions, Wasm memories, +/// Wasm globals, and Wasm tables. +/// +/// Used in [`ModuleInfo`] to access function signatures ([`SigIndex`]s, +/// [`FuncSig`]), [`GlobalInit`]s, [`MemoryDescriptor`]s, and +/// [`TableDescriptor`]s. +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Archive, RkyvSerialize, RkyvDeserialize)] +pub enum ExportIndex { + /// Function export index. [`FuncIndex`] is a type-safe handle referring to + /// a Wasm function. + Func(FuncIndex), + /// Memory export index. [`MemoryIndex`] is a type-safe handle referring to + /// a Wasm memory. + Memory(MemoryIndex), + /// Global export index. [`GlobalIndex`] is a type-safe handle referring to + /// a Wasm global. + Global(GlobalIndex), + /// Table export index. [`TableIndex`] is a type-safe handle referring to + /// to a Wasm table. + Table(TableIndex), +} + +/// A data initializer for linear memory. +#[derive(Serialize, Deserialize, Debug, Clone, Archive, RkyvSerialize, RkyvDeserialize)] +pub struct DataInitializer { + /// The index of the memory to initialize. + pub memory_index: MemoryIndex, + /// Either a constant offset or a `get_global` + pub base: Initializer, + /// The initialization data. + #[cfg_attr(feature = "cache", serde(with = "serde_bytes"))] + pub data: Vec, +} + +/// A WebAssembly table initializer. +#[derive(Serialize, Deserialize, Debug, Clone, Archive, RkyvSerialize, RkyvDeserialize)] +pub struct TableInitializer { + /// The index of a table to initialize. + pub table_index: TableIndex, + /// Either a constant offset or a `get_global` + pub base: Initializer, + /// The values to write into the table elements. + pub elements: Vec, +} + +/// String table builder. +#[derive(Archive, RkyvSerialize, RkyvDeserialize)] +pub struct StringTableBuilder { + map: IndexMap, + buffer: String, + count: u32, +} + +impl StringTableBuilder { + /// Creates a new [`StringTableBuilder`]. + pub fn new() -> Self { + Self { + map: IndexMap::new(), + buffer: String::new(), + count: 0, + } + } + + /// Register a new string into table. + pub fn register(&mut self, s: S) -> K + where + S: Into + AsRef, + { + let s_str = s.as_ref(); + + if self.map.contains_key(s_str) { + self.map[s_str].0 + } else { + let offset = self.buffer.len(); + let length = s_str.len(); + let index = TypedIndex::new(self.count as _); + + self.buffer.push_str(s_str); + self.map + .insert(s.into(), (index, offset as u32, length as u32)); + self.count += 1; + + index + } + } + + /// Finish building the [`StringTable`]. + pub fn finish(self) -> StringTable { + let table = self + .map + .values() + .map(|(_, offset, length)| (*offset, *length)) + .collect(); + + StringTable { + table, + buffer: self.buffer, + } + } +} + +/// A map of index to string. +#[derive(Serialize, Deserialize, Debug, Clone, Archive, RkyvSerialize, RkyvDeserialize)] +pub struct StringTable { + table: Map, + buffer: String, +} + +impl StringTable { + /// Creates a `StringTable`. + pub fn new() -> Self { + Self { + table: Map::new(), + buffer: String::new(), + } + } + + /// Gets a reference to a string at the given index. + pub fn get(&self, index: K) -> &str { + let (offset, length) = self.table[index]; + let offset = offset as usize; + let length = length as usize; + + &self.buffer[offset..offset + length] + } + + /// Gets all the strings in the StringTable as a Vec. + pub fn to_vec(&self) -> Vec<&str> { + self.table + .values() + .map(|(offset, length)| + &self.buffer[(*offset as usize)..(*offset as usize) + (*length as usize)]) + .collect() + } +} + +/// A type-safe handle referring to a module namespace. +#[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialEq, Eq, Hash, Archive, RkyvSerialize, RkyvDeserialize)] +pub struct NamespaceIndex(u32); + +impl TypedIndex for NamespaceIndex { + #[doc(hidden)] + fn new(index: usize) -> Self { + NamespaceIndex(index as _) + } + + #[doc(hidden)] + fn index(&self) -> usize { + self.0 as usize + } +} + +/// A type-safe handle referring to a name in a module namespace. +#[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialEq, Eq, Hash, Archive, RkyvSerialize, RkyvDeserialize)] +pub struct NameIndex(u32); + +impl TypedIndex for NameIndex { + #[doc(hidden)] + fn new(index: usize) -> Self { + NameIndex(index as _) + } + + #[doc(hidden)] + fn index(&self) -> usize { + self.0 as usize + } +} diff --git a/lib/runtime-core/src/parse.rs b/lib/runtime-core/src/parse.rs new file mode 100644 index 000000000000..8f042e4272ae --- /dev/null +++ b/lib/runtime-core/src/parse.rs @@ -0,0 +1,525 @@ +//! The parse module contains common data structures and functions using to parse wasm files into +//! runtime data structures. + +use crate::codegen::*; +use crate::{ + backend::{CompilerConfig, RunnableModule}, + error::CompileError, + module::{ + DataInitializer, ExportIndex, ImportName, ModuleInfo, StringTable, StringTableBuilder, + TableInitializer, + }, + structures::{Map, TypedIndex}, + types::{ + ElementType, FuncIndex, FuncSig, GlobalDescriptor, GlobalIndex, GlobalInit, + ImportedGlobalIndex, Initializer, MemoryDescriptor, MemoryIndex, SigIndex, TableDescriptor, + TableIndex, Type, Value, + }, + units::Pages, +}; +use std::collections::HashMap; +use std::fmt::Debug; +use std::sync::{Arc, RwLock}; +use wasmparser::{ + BinaryReaderError, ElemSectionEntryTable, ElementItem, ExternalKind, FuncType, + ImportSectionEntryType, Operator, Type as WpType, WasmDecoder, +}; + +/// Kind of load error. +#[derive(Debug)] +pub enum LoadError { + /// Parse error. + Parse(String), + /// Code generation error. + Codegen(String), +} + +impl From for CompileError { + fn from(other: LoadError) -> CompileError { + CompileError::InternalError { + msg: format!("{:?}", other), + } + } +} + +impl From for LoadError { + fn from(other: BinaryReaderError) -> LoadError { + LoadError::Parse(format!("{:?}", other)) + } +} + +impl From<&BinaryReaderError> for LoadError { + fn from(other: &BinaryReaderError) -> LoadError { + LoadError::Parse(format!("{:?}", other)) + } +} + +/// Read wasm binary into module data using the given backend, module code generator, middlewares, +/// and compiler configuration. +pub fn read_module< + MCG: ModuleCodeGenerator, + FCG: FunctionCodeGenerator, + RM: RunnableModule, + E: Debug, +>( + wasm: &[u8], + mcg: &mut MCG, + middlewares: &mut MiddlewareChain, + compiler_config: &CompilerConfig, +) -> Result>, LoadError> { + mcg.feed_compiler_config(compiler_config) + .map_err(|x| LoadError::Codegen(format!("{:?}", x)))?; + let info = Arc::new(RwLock::new(ModuleInfo { + memories: Map::new(), + globals: Map::new(), + tables: Map::new(), + + imported_functions: Map::new(), + imported_memories: Map::new(), + imported_tables: Map::new(), + imported_globals: Map::new(), + + exports: Default::default(), + + data_initializers: Vec::new(), + elem_initializers: Vec::new(), + + start_func: None, + + func_assoc: Map::new(), + signatures: Map::new(), + backend: MCG::backend_id().to_string(), + + namespace_table: StringTable::new(), + name_table: StringTable::new(), + + em_symbol_map: compiler_config.symbol_map.clone(), + + custom_sections: HashMap::new(), + + generate_debug_info: compiler_config.should_generate_debug_info(), + #[cfg(feature = "generate-debug-information")] + debug_info_manager: crate::jit_debug::JitCodeDebugInfoManager::new(), + })); + + let mut parser = wasmparser::ValidatingParser::new( + wasm, + Some(validating_parser_config(&compiler_config.features)), + ); + + let mut namespace_builder = Some(StringTableBuilder::new()); + let mut name_builder = Some(StringTableBuilder::new()); + let mut func_count: usize = 0; + let mut mcg_info_fed = false; + + loop { + use wasmparser::ParserState; + let state = parser.read(); + match *state { + ParserState::Error(ref err) => return Err(err.clone().into()), + ParserState::TypeSectionEntry(ref ty) => { + info.write() + .unwrap() + .signatures + .push(func_type_to_func_sig(ty)?); + } + ParserState::ImportSectionEntry { module, field, ty } => { + let namespace_index = namespace_builder.as_mut().unwrap().register(module); + let name_index = name_builder.as_mut().unwrap().register(field); + let import_name = ImportName { + namespace_index, + name_index, + }; + + match ty { + ImportSectionEntryType::Function(sigindex) => { + let sigindex = SigIndex::new(sigindex as usize); + info.write().unwrap().imported_functions.push(import_name); + info.write().unwrap().func_assoc.push(sigindex); + mcg.feed_import_function() + .map_err(|x| LoadError::Codegen(format!("{:?}", x)))?; + } + ImportSectionEntryType::Table(table_ty) => { + assert_eq!(table_ty.element_type, WpType::AnyFunc); + let table_desc = TableDescriptor { + element: ElementType::Anyfunc, + minimum: table_ty.limits.initial, + maximum: table_ty.limits.maximum, + }; + + info.write() + .unwrap() + .imported_tables + .push((import_name, table_desc)); + } + ImportSectionEntryType::Memory(memory_ty) => { + let mem_desc = MemoryDescriptor::new( + Pages(memory_ty.limits.initial), + memory_ty.limits.maximum.map(Pages), + memory_ty.shared, + ) + .map_err(|x| LoadError::Codegen(format!("{:?}", x)))?; + + info.write() + .unwrap() + .imported_memories + .push((import_name, mem_desc)); + } + ImportSectionEntryType::Global(global_ty) => { + let global_desc = GlobalDescriptor { + mutable: global_ty.mutable, + ty: wp_type_to_type(global_ty.content_type)?, + }; + info.write() + .unwrap() + .imported_globals + .push((import_name, global_desc)); + } + } + } + ParserState::FunctionSectionEntry(sigindex) => { + let sigindex = SigIndex::new(sigindex as usize); + info.write().unwrap().func_assoc.push(sigindex); + } + ParserState::TableSectionEntry(table_ty) => { + let table_desc = TableDescriptor { + element: ElementType::Anyfunc, + minimum: table_ty.limits.initial, + maximum: table_ty.limits.maximum, + }; + + info.write().unwrap().tables.push(table_desc); + } + ParserState::MemorySectionEntry(memory_ty) => { + let mem_desc = MemoryDescriptor::new( + Pages(memory_ty.limits.initial), + memory_ty.limits.maximum.map(Pages), + memory_ty.shared, + ) + .map_err(|x| LoadError::Codegen(format!("{:?}", x)))?; + + info.write().unwrap().memories.push(mem_desc); + } + ParserState::ExportSectionEntry { field, kind, index } => { + let export_index = match kind { + ExternalKind::Function => ExportIndex::Func(FuncIndex::new(index as usize)), + ExternalKind::Table => ExportIndex::Table(TableIndex::new(index as usize)), + ExternalKind::Memory => ExportIndex::Memory(MemoryIndex::new(index as usize)), + ExternalKind::Global => ExportIndex::Global(GlobalIndex::new(index as usize)), + }; + + info.write() + .unwrap() + .exports + .insert(field.to_string(), export_index); + } + ParserState::StartSectionEntry(start_index) => { + info.write().unwrap().start_func = Some(FuncIndex::new(start_index as usize)); + } + ParserState::BeginFunctionBody { range } => { + let id = func_count; + if !mcg_info_fed { + mcg_info_fed = true; + { + let mut info_write = info.write().unwrap(); + info_write.namespace_table = namespace_builder.take().unwrap().finish(); + info_write.name_table = name_builder.take().unwrap().finish(); + } + let info_read = info.read().unwrap(); + mcg.feed_signatures(info_read.signatures.clone()) + .map_err(|x| LoadError::Codegen(format!("{:?}", x)))?; + mcg.feed_function_signatures(info_read.func_assoc.clone()) + .map_err(|x| LoadError::Codegen(format!("{:?}", x)))?; + mcg.check_precondition(&info_read) + .map_err(|x| LoadError::Codegen(format!("{:?}", x)))?; + } + + let fcg = mcg + .next_function( + Arc::clone(&info), + WasmSpan::new(range.start as u32, range.end as u32), + ) + .map_err(|x| LoadError::Codegen(format!("{:?}", x)))?; + + { + let info_read = info.read().unwrap(); + let sig = info_read + .signatures + .get( + *info + .read() + .unwrap() + .func_assoc + .get(FuncIndex::new( + id as usize + info_read.imported_functions.len(), + )) + .unwrap(), + ) + .unwrap(); + for ret in sig.returns() { + fcg.feed_return(type_to_wp_type(*ret)) + .map_err(|x| LoadError::Codegen(format!("{:?}", x)))?; + } + for param in sig.params() { + fcg.feed_param(type_to_wp_type(*param)) + .map_err(|x| LoadError::Codegen(format!("{:?}", x)))?; + } + } + + let info_read = info.read().unwrap(); + let mut cur_pos = parser.current_position() as u32; + let mut state = parser.read(); + // loop until the function body starts + loop { + match state { + ParserState::Error(err) => return Err(err.into()), + ParserState::FunctionBodyLocals { ref locals } => { + for &(count, ty) in locals.iter() { + fcg.feed_local(ty, count as usize, cur_pos) + .map_err(|x| LoadError::Codegen(format!("{:?}", x)))?; + middlewares.run_func_local(ty, count as usize, cur_pos) + .map_err(|x| LoadError::Codegen(format!("{:?}", x)))?; + } + } + ParserState::CodeOperator(_) => { + // the body of the function has started + fcg.begin_body(&info_read) + .map_err(|x| LoadError::Codegen(format!("{:?}", x)))?; + middlewares + .run( + Some(fcg), + Event::Internal(InternalEvent::FunctionBegin(id as u32)), + &info_read, + cur_pos, + ) + .map_err(LoadError::Codegen)?; + // go to other loop + break; + } + ParserState::EndFunctionBody => break, + _ => unreachable!(), + } + cur_pos = parser.current_position() as u32; + state = parser.read(); + } + + // loop until the function body ends + loop { + match state { + ParserState::Error(err) => return Err(err.into()), + ParserState::CodeOperator(op) => { + middlewares + .run(Some(fcg), Event::Wasm(op), &info_read, cur_pos) + .map_err(LoadError::Codegen)?; + } + ParserState::EndFunctionBody => break, + _ => unreachable!(), + } + cur_pos = parser.current_position() as u32; + state = parser.read(); + } + middlewares + .run( + Some(fcg), + Event::Internal(InternalEvent::FunctionEnd), + &info_read, + cur_pos, + ) + .map_err(LoadError::Codegen)?; + + fcg.finalize() + .map_err(|x| LoadError::Codegen(format!("{:?}", x)))?; + func_count = func_count.wrapping_add(1); + } + ParserState::BeginElementSectionEntry { + table: ElemSectionEntryTable::Active(table_index_raw), + ty: WpType::AnyFunc, + } => { + let table_index = TableIndex::new(table_index_raw as usize); + let mut elements: Option> = None; + let mut base: Option = None; + + loop { + let state = parser.read(); + match *state { + ParserState::Error(ref err) => return Err(err.into()), + ParserState::InitExpressionOperator(ref op) => { + base = Some(eval_init_expr(op)?) + } + ParserState::ElementSectionEntryBody(ref _elements) => { + elements = Some( + _elements + .iter() + .map(|elem_idx| match elem_idx { + ElementItem::Null => Err(LoadError::Parse(format!("Error at table {}: null entries in tables are not yet supported", table_index_raw))), + ElementItem::Func(idx) => Ok(FuncIndex::new(*idx as usize)), + }) + .collect::, LoadError>>()?, + ); + } + ParserState::BeginInitExpressionBody + | ParserState::EndInitExpressionBody => {} + ParserState::EndElementSectionEntry => break, + _ => unreachable!(), + } + } + + let table_init = TableInitializer { + table_index, + base: base.unwrap(), + elements: elements.unwrap(), + }; + + info.write().unwrap().elem_initializers.push(table_init); + } + ParserState::BeginElementSectionEntry { + table: ElemSectionEntryTable::Active(table_index), + ty, + } => { + return Err(LoadError::Parse(format!( + "Error at table {}: type \"{:?}\" is not supported in tables yet", + table_index, ty + ))); + } + ParserState::BeginActiveDataSectionEntry(memory_index) => { + let memory_index = MemoryIndex::new(memory_index as usize); + let mut base: Option = None; + let mut data: Vec = vec![]; + + loop { + let state = parser.read(); + match *state { + ParserState::Error(ref err) => return Err(err.into()), + ParserState::InitExpressionOperator(ref op) => { + base = Some(eval_init_expr(op)?) + } + ParserState::DataSectionEntryBodyChunk(chunk) => { + data.extend_from_slice(chunk); + } + ParserState::BeginInitExpressionBody + | ParserState::EndInitExpressionBody => {} + ParserState::BeginDataSectionEntryBody(_) + | ParserState::EndDataSectionEntryBody => {} + ParserState::EndDataSectionEntry => break, + _ => unreachable!(), + } + } + + let data_init = DataInitializer { + memory_index, + base: base.unwrap(), + data, + }; + info.write().unwrap().data_initializers.push(data_init); + } + ParserState::BeginGlobalSectionEntry(ty) => { + let init = loop { + let state = parser.read(); + match *state { + ParserState::Error(ref err) => return Err(err.into()), + ParserState::InitExpressionOperator(ref op) => { + break eval_init_expr(op)?; + } + ParserState::BeginInitExpressionBody => {} + _ => unreachable!(), + } + }; + let desc = GlobalDescriptor { + mutable: ty.mutable, + ty: wp_type_to_type(ty.content_type)?, + }; + + let global_init = GlobalInit { desc, init }; + + info.write().unwrap().globals.push(global_init); + } + ParserState::EndWasm => { + // TODO Consolidate with BeginFunction body if possible + if !mcg_info_fed { + info.write().unwrap().namespace_table = + namespace_builder.take().unwrap().finish(); + info.write().unwrap().name_table = name_builder.take().unwrap().finish(); + mcg.feed_signatures(info.read().unwrap().signatures.clone()) + .map_err(|x| LoadError::Codegen(format!("{:?}", x)))?; + mcg.feed_function_signatures(info.read().unwrap().func_assoc.clone()) + .map_err(|x| LoadError::Codegen(format!("{:?}", x)))?; + mcg.check_precondition(&info.read().unwrap()) + .map_err(|x| LoadError::Codegen(format!("{:?}", x)))?; + } + break; + } + _ => {} + } + } + Ok(info) +} + +/// Convert given `WpType` to `Type`. +pub fn wp_type_to_type(ty: WpType) -> Result { + match ty { + WpType::I32 => Ok(Type::I32), + WpType::I64 => Ok(Type::I64), + WpType::F32 => Ok(Type::F32), + WpType::F64 => Ok(Type::F64), + WpType::V128 => Ok(Type::V128), + _ => { + return Err(LoadError::Parse( + "broken invariant, invalid type".to_string(), + )); + } + } +} + +/// Convert given `Type` to `WpType`. +pub fn type_to_wp_type(ty: Type) -> WpType { + match ty { + Type::I32 => WpType::I32, + Type::I64 => WpType::I64, + Type::F32 => WpType::F32, + Type::F64 => WpType::F64, + Type::V128 => WpType::V128, + } +} + +fn func_type_to_func_sig(func_ty: &FuncType) -> Result { + assert_eq!(func_ty.form, WpType::Func); + + Ok(FuncSig::new( + func_ty + .params + .iter() + .cloned() + .map(wp_type_to_type) + .collect::, _>>()?, + func_ty + .returns + .iter() + .cloned() + .map(wp_type_to_type) + .collect::, _>>()?, + )) +} + +fn eval_init_expr(op: &Operator) -> Result { + Ok(match *op { + Operator::GlobalGet { global_index } => { + Initializer::GetGlobal(ImportedGlobalIndex::new(global_index as usize)) + } + Operator::I32Const { value } => Initializer::Const(Value::I32(value)), + Operator::I64Const { value } => Initializer::Const(Value::I64(value)), + Operator::F32Const { value } => { + Initializer::Const(Value::F32(f32::from_bits(value.bits()))) + } + Operator::F64Const { value } => { + Initializer::Const(Value::F64(f64::from_bits(value.bits()))) + } + Operator::V128Const { value } => { + Initializer::Const(Value::V128(u128::from_le_bytes(*value.bytes()))) + } + _ => { + return Err(LoadError::Parse( + "init expr evaluation failed: unsupported opcode".to_string(), + )); + } + }) +} diff --git a/lib/runtime-core/src/state.rs b/lib/runtime-core/src/state.rs new file mode 100644 index 000000000000..0041eb91983f --- /dev/null +++ b/lib/runtime-core/src/state.rs @@ -0,0 +1,1287 @@ +//! The state module is used to track state of a running web assembly instances so that +//! state could read or updated at runtime. Use cases include generating stack traces, switching +//! generated code from one tier to another, or serializing state of a running instace. + +use crate::backend::RunnableModule; +use std::collections::BTreeMap; +use std::ops::Bound::{Included, Unbounded}; +use std::sync::Arc; + +use rkyv::{ + Archive, + Serialize as RkyvSerialize, + Deserialize as RkyvDeserialize, +}; + +/// An index to a register +#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize, Archive, RkyvSerialize, RkyvDeserialize)] +pub struct RegisterIndex(pub usize); + +/// A kind of wasm or constant value +#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize, Archive, RkyvSerialize, RkyvDeserialize)] +pub enum WasmAbstractValue { + /// A wasm runtime value + Runtime, + /// A wasm constant value + Const(u64), +} + +/// A container for the state of a running wasm instance. +#[derive(Clone, Debug, Serialize, Deserialize, Archive, RkyvSerialize, RkyvDeserialize)] +pub struct MachineState { + /// Stack values. + pub stack_values: Vec, + /// Register values. + pub register_values: Vec, + /// Previous frame. + pub prev_frame: BTreeMap, + /// Wasm stack. + pub wasm_stack: Vec, + /// Private depth of the wasm stack. + pub wasm_stack_private_depth: usize, + /// Wasm instruction offset. + pub wasm_inst_offset: usize, +} + +/// A diff of two `MachineState`s. +#[derive(Clone, Debug, Default, Serialize, Deserialize, Archive, RkyvSerialize, RkyvDeserialize)] +pub struct MachineStateDiff { + /// Last. + pub last: Option, + /// Stack push. + pub stack_push: Vec, + /// Stack pop. + pub stack_pop: usize, + + /// Register diff. + pub reg_diff: Vec<(RegisterIndex, MachineValue)>, + + /// Previous frame diff. + pub prev_frame_diff: BTreeMap>, // None for removal + + /// Wasm stack push. + pub wasm_stack_push: Vec, + /// Wasm stack pop. + pub wasm_stack_pop: usize, + /// Private depth of the wasm stack. + pub wasm_stack_private_depth: usize, // absolute value; not a diff. + /// Wasm instruction offset. + pub wasm_inst_offset: usize, // absolute value; not a diff. +} + +/// A kind of machine value. +#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize, Archive, RkyvSerialize, RkyvDeserialize)] +pub enum MachineValue { + /// Undefined. + Undefined, + /// Vmctx. + Vmctx, + /// Vmctx Deref. + VmctxDeref(Vec), + /// Preserve Register. + PreserveRegister(RegisterIndex), + /// Copy Stack BP Relative. + CopyStackBPRelative(i32), // relative to Base Pointer, in byte offset + /// Explicit Shadow. + ExplicitShadow, // indicates that all values above this are above the shadow region + /// Wasm Stack. + WasmStack(usize), + /// Wasm Local. + WasmLocal(usize), + /// Two Halves. + TwoHalves(Box<(MachineSubvalue, MachineSubvalue)>), // 32-bit values. TODO: optimize: add another type for inner "half" value to avoid boxing? +} + +/// A kind of machine value used in MachineValue::TwoHalves. Created so that MachineValue does not +/// reference two more MachineValues as part of TwoHalves. +#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize, Archive, RkyvSerialize, RkyvDeserialize)] +pub enum MachineSubvalue { + /// Undefined. + Undefined, + /// Vmctx Deref. + VmctxDeref(Vec), + /// Wasm Stack. + WasmStack(usize), + /// Wasm Local. + WasmLocal(usize), +} + +impl From for MachineSubvalue { + #[inline] + fn from(subvalue: MachineValue) -> MachineSubvalue { + match subvalue { + MachineValue::Undefined => MachineSubvalue::Undefined, + MachineValue::VmctxDeref(v) => MachineSubvalue::VmctxDeref(v), + MachineValue::WasmStack(i) => MachineSubvalue::WasmStack(i), + MachineValue::WasmLocal(i) => MachineSubvalue::WasmLocal(i), + _ => unreachable!("invalid kind of MachineValue to convert to MachineSubvalue"), + } + } +} + +/// A map of function states. +#[derive(Clone, Debug, Serialize, Deserialize, Archive, RkyvSerialize, RkyvDeserialize)] +pub struct FunctionStateMap { + /// Initial. + pub initial: MachineState, + /// Local Function Id. + pub local_function_id: usize, + /// Locals. + pub locals: Vec, + /// Shadow size. + pub shadow_size: usize, // for single-pass backend, 32 bytes on x86-64 + /// Diffs. + pub diffs: Vec, + /// Wasm Function Header target offset. + pub wasm_function_header_target_offset: Option, + /// Wasm offset to target offset + pub wasm_offset_to_target_offset: BTreeMap, + /// Loop offsets. + pub loop_offsets: BTreeMap, /* suspend_offset -> info */ + /// Call offsets. + pub call_offsets: BTreeMap, /* suspend_offset -> info */ + /// Trappable offsets. + pub trappable_offsets: BTreeMap, /* suspend_offset -> info */ +} + +/// A kind of suspend offset. +#[derive(Clone, Copy, Debug, Serialize, Deserialize, Archive, RkyvSerialize, RkyvDeserialize)] +pub enum SuspendOffset { + /// A loop. + Loop(usize), + /// A call. + Call(usize), + /// A trappable. + Trappable(usize), +} + +/// Info for an offset. +#[derive(Clone, Debug, Serialize, Deserialize, Archive, RkyvSerialize, RkyvDeserialize)] +pub struct OffsetInfo { + /// End offset. + pub end_offset: usize, // excluded bound + /// Diff Id. + pub diff_id: usize, + /// Activate offset. + pub activate_offset: usize, +} + +/// A map of module state. +#[derive(Clone, Debug, Serialize, Deserialize, Archive, RkyvSerialize, RkyvDeserialize)] +pub struct ModuleStateMap { + /// Local functions. + pub local_functions: BTreeMap, + /// Total size. + pub total_size: usize, +} + +/// State dump of a wasm function. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct WasmFunctionStateDump { + /// Local function id. + pub local_function_id: usize, + /// Wasm instruction offset. + pub wasm_inst_offset: usize, + /// Stack. + pub stack: Vec>, + /// Locals. + pub locals: Vec>, +} + +/// An image of the execution state. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ExecutionStateImage { + /// Frames. + pub frames: Vec, +} + +/// Represents an image of an `Instance` including its memory, globals, and execution state. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct InstanceImage { + /// Memory for this `InstanceImage` + pub memory: Option>, + /// Stored globals for this `InstanceImage` + pub globals: Vec, + /// `ExecutionStateImage` for this `InstanceImage` + pub execution_state: ExecutionStateImage, +} + +/// A `CodeVersion` is a container for a unit of generated code for a module. +#[derive(Clone)] +pub struct CodeVersion { + /// Indicates if this code version is the baseline version. + pub baseline: bool, + + /// `ModuleStateMap` for this code version. + pub msm: ModuleStateMap, + + /// A pointer to the machine code for this module. + pub base: usize, + + /// The backend used to compile this module. + pub backend: &'static str, + + /// `RunnableModule` for this code version. + pub runnable_module: Arc>, +} + +impl ModuleStateMap { + /// Looks up an ip from self using the given ip, base, and offset table provider. + pub fn lookup_ip &BTreeMap>( + &self, + ip: usize, + base: usize, + offset_table_provider: F, + ) -> Option<(&FunctionStateMap, MachineState)> { + if ip < base || ip - base >= self.total_size { + None + } else { + let (_, fsm) = self + .local_functions + .range((Unbounded, Included(&(ip - base)))) + .last() + .unwrap(); + + match offset_table_provider(fsm) + .range((Unbounded, Included(&(ip - base)))) + .last() + { + Some((_, x)) => { + if ip - base >= x.end_offset { + None + } else if x.diff_id < fsm.diffs.len() { + Some((fsm, fsm.diffs[x.diff_id].build_state(fsm))) + } else { + None + } + } + None => None, + } + } + } + /// Looks up a call ip from self using the given ip and base values. + pub fn lookup_call_ip( + &self, + ip: usize, + base: usize, + ) -> Option<(&FunctionStateMap, MachineState)> { + self.lookup_ip(ip, base, |fsm| &fsm.call_offsets) + } + + /// Looks up a trappable ip from self using the given ip and base values. + pub fn lookup_trappable_ip( + &self, + ip: usize, + base: usize, + ) -> Option<(&FunctionStateMap, MachineState)> { + self.lookup_ip(ip, base, |fsm| &fsm.trappable_offsets) + } + + /// Looks up a loop ip from self using the given ip and base values. + pub fn lookup_loop_ip( + &self, + ip: usize, + base: usize, + ) -> Option<(&FunctionStateMap, MachineState)> { + self.lookup_ip(ip, base, |fsm| &fsm.loop_offsets) + } +} + +impl FunctionStateMap { + /// Creates a new `FunctionStateMap` with the given parameters. + pub fn new( + initial: MachineState, + local_function_id: usize, + shadow_size: usize, + locals: Vec, + ) -> FunctionStateMap { + FunctionStateMap { + initial, + local_function_id, + shadow_size, + locals, + diffs: vec![], + wasm_function_header_target_offset: None, + wasm_offset_to_target_offset: BTreeMap::new(), + loop_offsets: BTreeMap::new(), + call_offsets: BTreeMap::new(), + trappable_offsets: BTreeMap::new(), + } + } +} + +impl MachineState { + /// Creates a `MachineStateDiff` from self and the given `&MachineState`. + pub fn diff(&self, old: &MachineState) -> MachineStateDiff { + let first_diff_stack_depth: usize = self + .stack_values + .iter() + .zip(old.stack_values.iter()) + .enumerate() + .find(|&(_, (a, b))| a != b) + .map(|x| x.0) + .unwrap_or(old.stack_values.len().min(self.stack_values.len())); + assert_eq!(self.register_values.len(), old.register_values.len()); + let reg_diff: Vec<_> = self + .register_values + .iter() + .zip(old.register_values.iter()) + .enumerate() + .filter(|&(_, (a, b))| a != b) + .map(|(i, (a, _))| (RegisterIndex(i), a.clone())) + .collect(); + let prev_frame_diff: BTreeMap> = self + .prev_frame + .iter() + .filter(|(k, v)| { + if let Some(ref old_v) = old.prev_frame.get(k) { + v != old_v + } else { + true + } + }) + .map(|(&k, v)| (k, Some(v.clone()))) + .chain( + old.prev_frame + .iter() + .filter(|(k, _)| self.prev_frame.get(k).is_none()) + .map(|(&k, _)| (k, None)), + ) + .collect(); + let first_diff_wasm_stack_depth: usize = self + .wasm_stack + .iter() + .zip(old.wasm_stack.iter()) + .enumerate() + .find(|&(_, (a, b))| a != b) + .map(|x| x.0) + .unwrap_or(old.wasm_stack.len().min(self.wasm_stack.len())); + MachineStateDiff { + last: None, + stack_push: self.stack_values[first_diff_stack_depth..].to_vec(), + stack_pop: old.stack_values.len() - first_diff_stack_depth, + reg_diff, + + prev_frame_diff, + + wasm_stack_push: self.wasm_stack[first_diff_wasm_stack_depth..].to_vec(), + wasm_stack_pop: old.wasm_stack.len() - first_diff_wasm_stack_depth, + wasm_stack_private_depth: self.wasm_stack_private_depth, + + wasm_inst_offset: self.wasm_inst_offset, + } + } +} + +impl MachineStateDiff { + /// Creates a `MachineState` from the given `&FunctionStateMap`. + pub fn build_state(&self, m: &FunctionStateMap) -> MachineState { + let mut chain: Vec<&MachineStateDiff> = vec![]; + chain.push(self); + let mut current = self.last; + while let Some(x) = current { + let that = &m.diffs[x]; + current = that.last; + chain.push(that); + } + chain.reverse(); + let mut state = m.initial.clone(); + for x in chain { + for _ in 0..x.stack_pop { + state.stack_values.pop().unwrap(); + } + for v in &x.stack_push { + state.stack_values.push(v.clone()); + } + for &(index, ref v) in &x.reg_diff { + state.register_values[index.0] = v.clone(); + } + for (index, ref v) in &x.prev_frame_diff { + if let Some(ref x) = v { + state.prev_frame.insert(*index, x.clone()); + } else { + state.prev_frame.remove(index).unwrap(); + } + } + for _ in 0..x.wasm_stack_pop { + state.wasm_stack.pop().unwrap(); + } + for v in &x.wasm_stack_push { + state.wasm_stack.push(*v); + } + } + state.wasm_stack_private_depth = self.wasm_stack_private_depth; + state.wasm_inst_offset = self.wasm_inst_offset; + state + } +} + +impl ExecutionStateImage { + /// Prints a backtrace if the `WASMER_BACKTRACE` environment variable is 1. + pub fn print_backtrace_if_needed(&self) { + use std::env; + + if let Ok(x) = env::var("WASMER_BACKTRACE") { + if x == "1" { + eprintln!("{}", self.output()); + return; + } + } + + eprintln!("Run with `WASMER_BACKTRACE=1` environment variable to display a backtrace."); + } + + /// Converts self into a `String`, used for display purposes. + pub fn output(&self) -> String { + fn join_strings(x: impl Iterator, sep: &str) -> String { + let mut ret = String::new(); + let mut first = true; + + for s in x { + if first { + first = false; + } else { + ret += sep; + } + ret += &s; + } + + ret + } + + fn format_optional_u64_sequence(x: &[Option]) -> String { + if x.is_empty() { + "(empty)".into() + } else { + join_strings( + x.iter().enumerate().map(|(i, x)| { + format!( + "[{}] = {}", + i, + x.map(|x| format!("{}", x)) + .unwrap_or_else(|| "?".to_string()) + ) + }), + ", ", + ) + } + } + + let mut ret = String::new(); + + if self.frames.is_empty() { + ret += &"Unknown fault address, cannot read stack."; + ret += "\n"; + } else { + ret += &"Backtrace:"; + ret += "\n"; + for (i, f) in self.frames.iter().enumerate() { + ret += &format!("* Frame {} @ Local function {}", i, f.local_function_id); + ret += "\n"; + ret += &format!(" {} {}\n", "Offset:", format!("{}", f.wasm_inst_offset),); + ret += &format!( + " {} {}\n", + "Locals:", + format_optional_u64_sequence(&f.locals) + ); + ret += &format!( + " {} {}\n\n", + "Stack:", + format_optional_u64_sequence(&f.stack) + ); + } + } + + ret + } +} + +impl InstanceImage { + /// Converts a slice of bytes into an `Option` + pub fn from_bytes(input: &[u8]) -> Option { + use bincode::deserialize; + match deserialize(input) { + Ok(x) => Some(x), + Err(_) => None, + } + } + + /// Converts self into a vector of bytes. + pub fn to_bytes(&self) -> Vec { + use bincode::serialize; + serialize(self).unwrap() + } +} + +/// Declarations for x86-64 registers. +#[cfg(unix)] +pub mod x64_decl { + use super::*; + + /// General-purpose registers. + #[repr(u8)] + #[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] + pub enum GPR { + /// RAX register + RAX, + /// RCX register + RCX, + /// RDX register + RDX, + /// RBX register + RBX, + /// RSP register + RSP, + /// RBP register + RBP, + /// RSI register + RSI, + /// RDI register + RDI, + /// R8 register + R8, + /// R9 register + R9, + /// R10 register + R10, + /// R11 register + R11, + /// R12 register + R12, + /// R13 register + R13, + /// R14 register + R14, + /// R15 register + R15, + } + + /// XMM registers. + #[repr(u8)] + #[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] + pub enum XMM { + /// XMM register 0 + XMM0, + /// XMM register 1 + XMM1, + /// XMM register 2 + XMM2, + /// XMM register 3 + XMM3, + /// XMM register 4 + XMM4, + /// XMM register 5 + XMM5, + /// XMM register 6 + XMM6, + /// XMM register 7 + XMM7, + /// XMM register 8 + XMM8, + /// XMM register 9 + XMM9, + /// XMM register 10 + XMM10, + /// XMM register 11 + XMM11, + /// XMM register 12 + XMM12, + /// XMM register 13 + XMM13, + /// XMM register 14 + XMM14, + /// XMM register 15 + XMM15, + } + + /// A machine register under the x86-64 architecture. + #[derive(Copy, Clone, Debug, Eq, PartialEq)] + pub enum X64Register { + /// General-purpose registers. + GPR(GPR), + /// XMM (floating point/SIMD) registers. + XMM(XMM), + } + + impl X64Register { + /// Returns the index of the register. + pub fn to_index(&self) -> RegisterIndex { + match *self { + X64Register::GPR(x) => RegisterIndex(x as usize), + X64Register::XMM(x) => RegisterIndex(x as usize + 16), + } + } + + /// Converts a DWARD regnum to X64Register. + pub fn from_dwarf_regnum(x: u16) -> Option { + Some(match x { + 0 => X64Register::GPR(GPR::RAX), + 1 => X64Register::GPR(GPR::RDX), + 2 => X64Register::GPR(GPR::RCX), + 3 => X64Register::GPR(GPR::RBX), + 4 => X64Register::GPR(GPR::RSI), + 5 => X64Register::GPR(GPR::RDI), + 6 => X64Register::GPR(GPR::RBP), + 7 => X64Register::GPR(GPR::RSP), + 8 => X64Register::GPR(GPR::R8), + 9 => X64Register::GPR(GPR::R9), + 10 => X64Register::GPR(GPR::R10), + 11 => X64Register::GPR(GPR::R11), + 12 => X64Register::GPR(GPR::R12), + 13 => X64Register::GPR(GPR::R13), + 14 => X64Register::GPR(GPR::R14), + 15 => X64Register::GPR(GPR::R15), + + 17 => X64Register::XMM(XMM::XMM0), + 18 => X64Register::XMM(XMM::XMM1), + 19 => X64Register::XMM(XMM::XMM2), + 20 => X64Register::XMM(XMM::XMM3), + 21 => X64Register::XMM(XMM::XMM4), + 22 => X64Register::XMM(XMM::XMM5), + 23 => X64Register::XMM(XMM::XMM6), + 24 => X64Register::XMM(XMM::XMM7), + _ => return None, + }) + } + } +} + +#[cfg(unix)] +pub mod x64 { + //! The x64 state module contains functions to generate state and code for x64 targets. + pub use super::x64_decl::*; + use super::*; + use crate::codegen::BreakpointMap; + use crate::fault::{ + catch_unsafe_unwind, get_boundary_register_preservation, run_on_alternative_stack, + }; + use crate::structures::TypedIndex; + use crate::types::LocalGlobalIndex; + use crate::vm::Ctx; + use std::any::Any; + + #[allow(clippy::cast_ptr_alignment)] + unsafe fn compute_vmctx_deref(vmctx: *const Ctx, seq: &[usize]) -> u64 { + let mut ptr = &vmctx as *const *const Ctx as *const u8; + for x in seq { + debug_assert!(ptr.align_offset(std::mem::align_of::<*const u8>()) == 0); + ptr = (*(ptr as *const *const u8)).add(*x); + } + ptr as usize as u64 + } + + /// Create a new `MachineState` with default values. + pub fn new_machine_state() -> MachineState { + MachineState { + stack_values: vec![], + register_values: vec![MachineValue::Undefined; 16 + 8], + prev_frame: BTreeMap::new(), + wasm_stack: vec![], + wasm_stack_private_depth: 0, + wasm_inst_offset: ::std::usize::MAX, + } + } + + /// Invokes a call return on the stack for the given module state map, code base, instance + /// image and context. + #[warn(unused_variables)] + pub unsafe fn invoke_call_return_on_stack( + msm: &ModuleStateMap, + code_base: usize, + image: InstanceImage, + vmctx: &mut Ctx, + breakpoints: Option, + ) -> Result> { + let mut stack: Vec = vec![0; 1048576 * 8 / 8]; // 8MB stack + let mut stack_offset: usize = stack.len(); + + stack_offset -= 3; // placeholder for call return + + let mut last_stack_offset: u64 = 0; // rbp + + let mut known_registers: [Option; 32] = [None; 32]; + + let local_functions_vec: Vec<&FunctionStateMap> = + msm.local_functions.iter().map(|(_, v)| v).collect(); + + // Bottom to top + for f in image.execution_state.frames.iter().rev() { + let fsm = local_functions_vec[f.local_function_id]; + let suspend_offset = if f.wasm_inst_offset == ::std::usize::MAX { + fsm.wasm_function_header_target_offset + } else { + fsm.wasm_offset_to_target_offset + .get(&f.wasm_inst_offset) + .copied() + } + .expect("instruction is not a critical point"); + + let (activate_offset, diff_id) = match suspend_offset { + SuspendOffset::Loop(x) => fsm.loop_offsets.get(&x), + SuspendOffset::Call(x) => fsm.call_offsets.get(&x), + SuspendOffset::Trappable(x) => fsm.trappable_offsets.get(&x), + } + .map(|x| (x.activate_offset, x.diff_id)) + .expect("offset cannot be found in table"); + + let diff = &fsm.diffs[diff_id]; + let state = diff.build_state(fsm); + + stack_offset -= 1; + stack[stack_offset] = stack.as_ptr().offset(last_stack_offset as isize) as usize as u64; // push rbp + last_stack_offset = stack_offset as _; + + let mut got_explicit_shadow = false; + + for v in state.stack_values.iter() { + match *v { + MachineValue::Undefined => stack_offset -= 1, + MachineValue::Vmctx => { + stack_offset -= 1; + stack[stack_offset] = vmctx as *mut Ctx as usize as u64; + } + MachineValue::VmctxDeref(ref seq) => { + stack_offset -= 1; + stack[stack_offset] = compute_vmctx_deref(vmctx as *const Ctx, seq); + } + MachineValue::PreserveRegister(index) => { + stack_offset -= 1; + stack[stack_offset] = known_registers[index.0].unwrap_or(0); + } + MachineValue::CopyStackBPRelative(byte_offset) => { + assert!(byte_offset % 8 == 0); + let target_offset = (byte_offset / 8) as isize; + let v = stack[(last_stack_offset as isize + target_offset) as usize]; + stack_offset -= 1; + stack[stack_offset] = v; + } + MachineValue::ExplicitShadow => { + assert!(fsm.shadow_size % 8 == 0); + stack_offset -= fsm.shadow_size / 8; + got_explicit_shadow = true; + } + MachineValue::WasmStack(x) => { + stack_offset -= 1; + match state.wasm_stack[x] { + WasmAbstractValue::Const(x) => { + stack[stack_offset] = x; + } + WasmAbstractValue::Runtime => { + stack[stack_offset] = f.stack[x].unwrap(); + } + } + } + MachineValue::WasmLocal(x) => { + stack_offset -= 1; + match fsm.locals[x] { + WasmAbstractValue::Const(x) => { + stack[stack_offset] = x; + } + WasmAbstractValue::Runtime => { + stack[stack_offset] = f.locals[x].unwrap(); + } + } + } + MachineValue::TwoHalves(ref inner) => { + stack_offset -= 1; + // TODO: Cleanup + match inner.0 { + MachineSubvalue::WasmStack(x) => match state.wasm_stack[x] { + WasmAbstractValue::Const(x) => { + assert!(x <= std::u32::MAX as u64); + stack[stack_offset] |= x; + } + WasmAbstractValue::Runtime => { + let v = f.stack[x].unwrap(); + assert!(v <= std::u32::MAX as u64); + stack[stack_offset] |= v; + } + }, + MachineSubvalue::WasmLocal(x) => match fsm.locals[x] { + WasmAbstractValue::Const(x) => { + assert!(x <= std::u32::MAX as u64); + stack[stack_offset] |= x; + } + WasmAbstractValue::Runtime => { + let v = f.locals[x].unwrap(); + assert!(v <= std::u32::MAX as u64); + stack[stack_offset] |= v; + } + }, + MachineSubvalue::VmctxDeref(ref seq) => { + stack[stack_offset] |= + compute_vmctx_deref(vmctx as *const Ctx, seq) + & (std::u32::MAX as u64); + } + MachineSubvalue::Undefined => {} + } + match inner.1 { + MachineSubvalue::WasmStack(x) => match state.wasm_stack[x] { + WasmAbstractValue::Const(x) => { + assert!(x <= std::u32::MAX as u64); + stack[stack_offset] |= x << 32; + } + WasmAbstractValue::Runtime => { + let v = f.stack[x].unwrap(); + assert!(v <= std::u32::MAX as u64); + stack[stack_offset] |= v << 32; + } + }, + MachineSubvalue::WasmLocal(x) => match fsm.locals[x] { + WasmAbstractValue::Const(x) => { + assert!(x <= std::u32::MAX as u64); + stack[stack_offset] |= x << 32; + } + WasmAbstractValue::Runtime => { + let v = f.locals[x].unwrap(); + assert!(v <= std::u32::MAX as u64); + stack[stack_offset] |= v << 32; + } + }, + MachineSubvalue::VmctxDeref(ref seq) => { + stack[stack_offset] |= + (compute_vmctx_deref(vmctx as *const Ctx, seq) + & (std::u32::MAX as u64)) + << 32; + } + MachineSubvalue::Undefined => {} + } + } + } + } + if !got_explicit_shadow { + assert!(fsm.shadow_size % 8 == 0); + stack_offset -= fsm.shadow_size / 8; + } + for (i, v) in state.register_values.iter().enumerate() { + match *v { + MachineValue::Undefined => {} + MachineValue::Vmctx => { + known_registers[i] = Some(vmctx as *mut Ctx as usize as u64); + } + MachineValue::VmctxDeref(ref seq) => { + known_registers[i] = Some(compute_vmctx_deref(vmctx as *const Ctx, seq)); + } + MachineValue::WasmStack(x) => match state.wasm_stack[x] { + WasmAbstractValue::Const(x) => { + known_registers[i] = Some(x); + } + WasmAbstractValue::Runtime => { + known_registers[i] = Some(f.stack[x].unwrap()); + } + }, + MachineValue::WasmLocal(x) => match fsm.locals[x] { + WasmAbstractValue::Const(x) => { + known_registers[i] = Some(x); + } + WasmAbstractValue::Runtime => { + known_registers[i] = Some(f.locals[x].unwrap()); + } + }, + _ => unreachable!(), + } + } + + // no need to check 16-byte alignment here because it's possible that we're not at a call entry. + + stack_offset -= 1; + stack[stack_offset] = (code_base + activate_offset) as u64; // return address + } + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::R15).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::R14).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::R13).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::R12).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::R11).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::R10).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::R9).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::R8).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::RSI).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::RDI).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::RDX).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::RCX).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::RBX).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::RAX).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = stack.as_ptr().offset(last_stack_offset as isize) as usize as u64; // rbp + + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM15).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM14).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM13).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM12).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM11).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM10).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM9).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM8).to_index().0].unwrap_or(0); + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM7).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM6).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM5).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM4).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM3).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM2).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM1).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM0).to_index().0].unwrap_or(0); + + if let Some(ref memory) = image.memory { + assert!(vmctx.internal.memory_bound <= memory.len()); + + if vmctx.internal.memory_bound < memory.len() { + let grow: unsafe extern "C" fn(ctx: &mut Ctx, memory_index: usize, delta: usize) = + ::std::mem::transmute((*vmctx.internal.intrinsics).memory_grow); + grow( + vmctx, + 0, + (memory.len() - vmctx.internal.memory_bound) / 65536, + ); + assert_eq!(vmctx.internal.memory_bound, memory.len()); + } + + std::slice::from_raw_parts_mut(vmctx.internal.memory_base, vmctx.internal.memory_bound) + .copy_from_slice(memory); + } + + let globals_len = (*vmctx.module).info.globals.len(); + for i in 0..globals_len { + (*(*vmctx.local_backing).globals[LocalGlobalIndex::new(i)].vm_local_global()).data = + image.globals[i]; + } + + drop(image); // free up host memory + + catch_unsafe_unwind( + || { + run_on_alternative_stack( + stack.as_mut_ptr().add(stack.len()), + stack.as_mut_ptr().add(stack_offset), + ) + }, + breakpoints, + ) + } + + /// Builds an `InstanceImage` for the given `Ctx` and `ExecutionStateImage`. + pub fn build_instance_image( + vmctx: &mut Ctx, + execution_state: ExecutionStateImage, + ) -> InstanceImage { + unsafe { + let memory = if vmctx.internal.memory_base.is_null() { + None + } else { + Some( + std::slice::from_raw_parts( + vmctx.internal.memory_base, + vmctx.internal.memory_bound, + ) + .to_vec(), + ) + }; + + // FIXME: Imported globals + let globals_len = (*vmctx.module).info.globals.len(); + let globals: Vec = (0..globals_len) + .map(|i| { + (*vmctx.local_backing).globals[LocalGlobalIndex::new(i)] + .get() + .to_u128() + }) + .collect(); + + InstanceImage { + memory: memory, + globals: globals, + execution_state: execution_state, + } + } + } + + /// Returns a `ExecutionStateImage` for the given versions, stack, initial registers and + /// initial address. + #[warn(unused_variables)] + pub unsafe fn read_stack<'a, I: Iterator, F: Fn() -> I + 'a>( + versions: F, + mut stack: *const u64, + initially_known_registers: [Option; 32], + mut initial_address: Option, + max_depth: Option, + ) -> ExecutionStateImage { + let mut known_registers: [Option; 32] = initially_known_registers; + let mut results: Vec = vec![]; + let mut was_baseline = true; + + for depth in 0.. { + if let Some(max_depth) = max_depth { + if depth >= max_depth { + return ExecutionStateImage { frames: results }; + } + } + + let ret_addr = initial_address.take().unwrap_or_else(|| { + let x = *stack; + stack = stack.offset(1); + x + }); + + let mut fsm_state: Option<(&FunctionStateMap, MachineState)> = None; + let mut is_baseline: Option = None; + + for version in versions() { + match version + .msm + .lookup_call_ip(ret_addr as usize, version.base) + .or_else(|| { + version + .msm + .lookup_trappable_ip(ret_addr as usize, version.base) + }) + .or_else(|| version.msm.lookup_loop_ip(ret_addr as usize, version.base)) + { + Some(x) => { + fsm_state = Some(x); + is_baseline = Some(version.baseline); + break; + } + None => {} + }; + } + + let (fsm, state) = if let Some(x) = fsm_state { + x + } else { + return ExecutionStateImage { frames: results }; + }; + + { + let is_baseline = is_baseline.unwrap(); + + // Are we unwinding through an optimized/baseline boundary? + if is_baseline && !was_baseline { + let callee_saved = &*get_boundary_register_preservation(); + known_registers[X64Register::GPR(GPR::R15).to_index().0] = + Some(callee_saved.r15); + known_registers[X64Register::GPR(GPR::R14).to_index().0] = + Some(callee_saved.r14); + known_registers[X64Register::GPR(GPR::R13).to_index().0] = + Some(callee_saved.r13); + known_registers[X64Register::GPR(GPR::R12).to_index().0] = + Some(callee_saved.r12); + known_registers[X64Register::GPR(GPR::RBX).to_index().0] = + Some(callee_saved.rbx); + } + + was_baseline = is_baseline; + } + + let mut wasm_stack: Vec> = state + .wasm_stack + .iter() + .map(|x| match *x { + WasmAbstractValue::Const(x) => Some(x), + WasmAbstractValue::Runtime => None, + }) + .collect(); + let mut wasm_locals: Vec> = fsm + .locals + .iter() + .map(|x| match *x { + WasmAbstractValue::Const(x) => Some(x), + WasmAbstractValue::Runtime => None, + }) + .collect(); + + // This must be before the next loop because that modifies `known_registers`. + for (i, v) in state.register_values.iter().enumerate() { + match *v { + MachineValue::Undefined => {} + MachineValue::Vmctx => {} + MachineValue::VmctxDeref(_) => {} + MachineValue::WasmStack(idx) => { + if let Some(v) = known_registers[i] { + wasm_stack[idx] = Some(v); + } else { + eprintln!( + "BUG: Register {} for WebAssembly stack slot {} has unknown value.", + i, idx + ); + } + } + MachineValue::WasmLocal(idx) => { + if let Some(v) = known_registers[i] { + wasm_locals[idx] = Some(v); + } + } + _ => unreachable!(), + } + } + + let found_shadow = state + .stack_values + .iter() + .any(|v| *v == MachineValue::ExplicitShadow); + if !found_shadow { + stack = stack.add(fsm.shadow_size / 8); + } + + for v in state.stack_values.iter().rev() { + match *v { + MachineValue::ExplicitShadow => { + stack = stack.add(fsm.shadow_size / 8); + } + MachineValue::Undefined => { + stack = stack.offset(1); + } + MachineValue::Vmctx => { + stack = stack.offset(1); + } + MachineValue::VmctxDeref(_) => { + stack = stack.offset(1); + } + MachineValue::PreserveRegister(idx) => { + known_registers[idx.0] = Some(*stack); + stack = stack.offset(1); + } + MachineValue::CopyStackBPRelative(_) => { + stack = stack.offset(1); + } + MachineValue::WasmStack(idx) => { + wasm_stack[idx] = Some(*stack); + stack = stack.offset(1); + } + MachineValue::WasmLocal(idx) => { + wasm_locals[idx] = Some(*stack); + stack = stack.offset(1); + } + MachineValue::TwoHalves(ref inner) => { + let v = *stack; + stack = stack.offset(1); + match inner.0 { + MachineSubvalue::WasmStack(idx) => { + wasm_stack[idx] = Some(v & 0xffffffffu64); + } + MachineSubvalue::WasmLocal(idx) => { + wasm_locals[idx] = Some(v & 0xffffffffu64); + } + MachineSubvalue::VmctxDeref(_) => {} + MachineSubvalue::Undefined => {} + } + match inner.1 { + MachineSubvalue::WasmStack(idx) => { + wasm_stack[idx] = Some(v >> 32); + } + MachineSubvalue::WasmLocal(idx) => { + wasm_locals[idx] = Some(v >> 32); + } + MachineSubvalue::VmctxDeref(_) => {} + MachineSubvalue::Undefined => {} + } + } + } + } + + for (offset, v) in state.prev_frame.iter() { + let offset = (*offset + 2) as isize; // (saved_rbp, return_address) + match *v { + MachineValue::WasmStack(idx) => { + wasm_stack[idx] = Some(*stack.offset(offset)); + } + MachineValue::WasmLocal(idx) => { + wasm_locals[idx] = Some(*stack.offset(offset)); + } + _ => unreachable!("values in prev frame can only be stack/local"), + } + } + stack = stack.offset(1); // saved_rbp + + wasm_stack.truncate( + wasm_stack + .len() + .checked_sub(state.wasm_stack_private_depth) + .unwrap(), + ); + + let wfs = WasmFunctionStateDump { + local_function_id: fsm.local_function_id, + wasm_inst_offset: state.wasm_inst_offset, + stack: wasm_stack, + locals: wasm_locals, + }; + results.push(wfs); + } + + unreachable!(); + } +} diff --git a/lib/runtime-core/src/structures/map.rs b/lib/runtime-core/src/structures/map.rs new file mode 100644 index 000000000000..9e41fd75a4e0 --- /dev/null +++ b/lib/runtime-core/src/structures/map.rs @@ -0,0 +1,253 @@ +use super::{BoxedMap, SliceMap, TypedIndex}; +use std::{ + iter::{self, Extend, FromIterator}, + marker::PhantomData, + mem, + ops::{Deref, DerefMut}, + slice, vec, +}; + +use rkyv::{Archive, Serialize as RkyvSerialize, Deserialize as RkyvDeserialize}; + +/// Dense item map +#[derive(Serialize, Deserialize, Debug, Clone, Archive, RkyvSerialize, RkyvDeserialize)] +pub struct Map +where + K: TypedIndex, +{ + elems: Vec, + _marker: PhantomData, +} + +impl Map +where + K: TypedIndex, +{ + /// Creates a new `Map`. + pub fn new() -> Self { + Self { + elems: Vec::new(), + _marker: PhantomData, + } + } + + /// Creates a new empty `Map` with the given capacity. + pub fn with_capacity(capacity: usize) -> Self { + Self { + elems: Vec::with_capacity(capacity), + _marker: PhantomData, + } + } + + /// Clears the map. Keeps the allocated memory for future use. + pub fn clear(&mut self) { + self.elems.clear(); + } + + /// Returns the size of this map. + pub fn len(&self) -> usize { + self.elems.len() + } + + /// Returns true if this map is empty. + pub fn is_empty(&self) -> bool { + self.elems.is_empty() + } + + /// Adds a new value to this map. + pub fn push(&mut self, value: V) -> K { + let len = self.len(); + self.elems.push(value); + K::new(len) + } + + /// Returns the next index into the map. + pub fn next_index(&self) -> K { + K::new(self.len()) + } + + /// Reserves the given size. + pub fn reserve_exact(&mut self, size: usize) { + self.elems.reserve_exact(size); + } + + /// Convert this into a `BoxedMap`. + pub fn into_boxed_map(self) -> BoxedMap { + BoxedMap::new(self.elems.into_boxed_slice()) + } + + /// Convert this into a `Vec`. + pub fn into_vec(self) -> Vec { + self.elems + } + + /// Iterate over the values of the map in order. + pub fn values(&self) -> impl Iterator { + self.elems.iter() + } +} + +impl Map +where + K: TypedIndex, + V: Clone, +{ + /// Resize this map to the given new length and value. + pub fn resize(&mut self, new_len: usize, value: V) { + self.elems.resize(new_len, value); + } +} + +impl Extend for Map +where + K: TypedIndex, +{ + fn extend>(&mut self, iter: I) { + self.elems.extend(iter); + } +} + +impl FromIterator for Map +where + K: TypedIndex, +{ + fn from_iter>(iter: I) -> Self { + let elems: Vec = iter.into_iter().collect(); + Self { + elems, + _marker: PhantomData, + } + } +} + +impl Deref for Map +where + K: TypedIndex, +{ + type Target = SliceMap; + fn deref(&self) -> &SliceMap { + unsafe { mem::transmute::<&[V], _>(self.elems.as_slice()) } + } +} + +impl DerefMut for Map +where + K: TypedIndex, +{ + fn deref_mut(&mut self) -> &mut SliceMap { + unsafe { mem::transmute::<&mut [V], _>(self.elems.as_mut_slice()) } + } +} + +pub struct IntoIter +where + K: TypedIndex, +{ + enumerated: iter::Enumerate>, + _marker: PhantomData, +} + +impl IntoIter +where + K: TypedIndex, +{ + pub(in crate::structures) fn new(into_iter: vec::IntoIter) -> Self { + Self { + enumerated: into_iter.enumerate(), + _marker: PhantomData, + } + } +} + +impl Iterator for IntoIter +where + K: TypedIndex, +{ + type Item = (K, V); + + fn next(&mut self) -> Option<(K, V)> { + self.enumerated.next().map(|(i, v)| (K::new(i), v)) + } +} + +impl IntoIterator for Map +where + K: TypedIndex, +{ + type Item = (K, V); + type IntoIter = IntoIter; + + fn into_iter(self) -> Self::IntoIter { + IntoIter::new(self.elems.into_iter()) + } +} + +impl<'a, K, V> IntoIterator for &'a Map +where + K: TypedIndex, +{ + type Item = (K, &'a V); + type IntoIter = Iter<'a, K, V>; + + fn into_iter(self) -> Self::IntoIter { + Iter::new(self.elems.iter()) + } +} + +impl<'a, K, V> IntoIterator for &'a mut Map +where + K: TypedIndex, +{ + type Item = (K, &'a mut V); + type IntoIter = IterMut<'a, K, V>; + + fn into_iter(self) -> Self::IntoIter { + IterMut::new(self.elems.iter_mut()) + } +} + +/// Iterator for a `Map`. +pub struct Iter<'a, K: TypedIndex, V: 'a> { + enumerated: iter::Enumerate>, + _marker: PhantomData, +} + +impl<'a, K: TypedIndex, V: 'a> Iter<'a, K, V> { + pub(in crate::structures) fn new(iter: slice::Iter<'a, V>) -> Self { + Self { + enumerated: iter.enumerate(), + _marker: PhantomData, + } + } +} + +impl<'a, K: TypedIndex, V: 'a> Iterator for Iter<'a, K, V> { + type Item = (K, &'a V); + + fn next(&mut self) -> Option { + self.enumerated.next().map(|(i, v)| (K::new(i), v)) + } +} + +/// Mutable iterator for a `Map`. +pub struct IterMut<'a, K: TypedIndex, V: 'a> { + enumerated: iter::Enumerate>, + _marker: PhantomData, +} + +impl<'a, K: TypedIndex, V: 'a> IterMut<'a, K, V> { + pub(in crate::structures) fn new(iter: slice::IterMut<'a, V>) -> Self { + Self { + enumerated: iter.enumerate(), + _marker: PhantomData, + } + } +} + +impl<'a, K: TypedIndex, V: 'a> Iterator for IterMut<'a, K, V> { + type Item = (K, &'a mut V); + + fn next(&mut self) -> Option { + self.enumerated.next().map(|(i, v)| (K::new(i), v)) + } +} diff --git a/lib/runtime-core/src/sys/memory_rkyv.rs b/lib/runtime-core/src/sys/memory_rkyv.rs new file mode 100644 index 000000000000..c7954eb77358 --- /dev/null +++ b/lib/runtime-core/src/sys/memory_rkyv.rs @@ -0,0 +1,149 @@ +#[cfg(unix)] +use crate::sys::unix::{Memory, Protect}; + +#[cfg(windows)] +use crate::sys::windows::{Memory, Protect}; + +use rkyv::{ + Archive, + Archived, + Fallible, + Serialize as RkyvSerialize, + Deserialize as RkyvDeserialize, + ser::{Serializer, ScratchSpace}, + with::{ArchiveWith, SerializeWith, DeserializeWith}, +}; + +/// A serializable wrapper for Memory. +pub struct ArchivableMemory; + +/// The archived contents of a Memory. +#[derive(Archive, RkyvSerialize, RkyvDeserialize, Debug, PartialEq)] +#[archive(compare(PartialEq))] +#[archive_attr(derive(Debug))] +#[archive_attr(derive(PartialEq))] +pub struct CompactMemory { + contents: Vec, + content_size: u32, + protection: Protect, +} + +impl CompactMemory { + /// Construct a CompactMemory from a Memory. + pub unsafe fn from_memory(memory: &Memory) -> Self { + CompactMemory { + contents: memory.as_slice().to_vec(), + content_size: memory.content_size(), + protection: memory.protection(), + } + } + + /// Construct a Memory from a CompactMemory. + pub unsafe fn into_memory(&self) -> Memory { + let bytes = self.contents.as_slice(); + + let mut memory = Memory::with_size_protect(bytes.len(), Protect::ReadWrite) + .expect("Could not create a memory"); + + memory.as_slice_mut().copy_from_slice(&*bytes); + + if memory.protection() != self.protection { + memory + .protect(.., self.protection) + .expect("Could not protect memory as its original protection"); + } + + memory.set_content_size(self.content_size); + + memory + } +} + +impl ArchiveWith for ArchivableMemory { + type Archived = ::Archived; + type Resolver = ::Resolver; + + unsafe fn resolve_with(memory: &Memory, pos: usize, resolver: Self::Resolver, out: *mut Self::Archived) { + let archived_memory = CompactMemory::from_memory(memory); + archived_memory.resolve(pos, resolver, out); + } +} + +impl SerializeWith for ArchivableMemory +where + S: Serializer + ScratchSpace +{ + fn serialize_with(memory: &Memory, serializer: &mut S) -> Result { + unsafe { + let archived_memory = CompactMemory::from_memory(memory); + archived_memory.serialize(serializer) + } + } +} + +impl DeserializeWith, Memory, D> for ArchivableMemory +{ + fn deserialize_with(archived_memory: &Archived, deserializer: &mut D) -> Result { + let compact_memory: CompactMemory = archived_memory.deserialize(deserializer)?; + let memory = unsafe { compact_memory.into_memory() }; + + Ok(memory) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rkyv::ser::serializers::AllocSerializer; + use crate::sys::unix::*; + + #[test] + fn test_new_memory() { + let bytes = make_test_bytes(); + let memory = make_test_memory(&bytes); + unsafe { + assert_eq!(memory.as_slice(), bytes); + } + } + + #[test] + fn test_rkyv_compact_memory() { + let bytes = make_test_bytes(); + let memory = make_test_memory(&bytes); + + let compact_memory = unsafe { CompactMemory::from_memory(&memory) }; + + let mut serializer = AllocSerializer::<4096>::default(); + serializer.serialize_value(&compact_memory).unwrap(); + let serialized = serializer.into_serializer().into_inner(); + assert!(serialized.len() > 0); + + let archived = unsafe { rkyv::archived_root::(&serialized[..]) }; + + let deserialized: CompactMemory = archived.deserialize(&mut rkyv::Infallible).unwrap(); + assert_eq!(deserialized, compact_memory); + + let deserialized_memory = unsafe { deserialized.into_memory() }; + assert_eq!(deserialized_memory.protection(), memory.protection()); + unsafe { + assert_eq!(deserialized_memory.as_slice(), memory.as_slice()); + }; + } + + fn make_test_memory(bytes: &Vec) -> Memory { + let mut memory = Memory::with_size_protect(1000, Protect::ReadWrite) + .expect("Could not create memory"); + unsafe { + memory.as_slice_mut().copy_from_slice(&bytes[..]); + } + memory + } + + fn make_test_bytes() -> Vec { + let page_size = page_size::get(); + let mut bytes = b"abcdefghijkl".to_vec(); + let padding_zeros = [0 as u8; 1].repeat(page_size - bytes.len()); + bytes.extend(padding_zeros); + bytes + } +} diff --git a/lib/runtime-core/src/sys/mod.rs b/lib/runtime-core/src/sys/mod.rs new file mode 100644 index 000000000000..19a041f5ee26 --- /dev/null +++ b/lib/runtime-core/src/sys/mod.rs @@ -0,0 +1,98 @@ +mod memory_rkyv; + +#[cfg(unix)] +mod unix; + +#[cfg(windows)] +mod windows; + +#[cfg(unix)] +pub use self::unix::*; + +#[cfg(windows)] +pub use self::windows::*; + +pub use self::memory_rkyv::*; + +use serde::{ + de::{self, SeqAccess, Visitor}, + ser::SerializeStruct, + Deserialize, Deserializer, Serialize, Serializer, +}; + +use serde_bytes::{ByteBuf, Bytes}; + +use std::fmt; + +impl Serialize for Memory { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + assert!(self.protection().is_readable()); + + let mut state = serializer.serialize_struct("Memory", 2)?; + state.serialize_field("protection", &self.protection())?; + state.serialize_field("data", &Bytes::new(unsafe { self.as_slice() }))?; + state.end() + } +} + +impl<'de> Deserialize<'de> for Memory { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct MemoryVisitor; + + impl<'de> Visitor<'de> for MemoryVisitor { + type Value = Memory; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "struct Memory") + } + + fn visit_seq(self, mut seq: V) -> Result + where + V: SeqAccess<'de>, + { + let original_protection = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(0, &self))?; + + let bytes: ByteBuf = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(1, &self))?; + + let mut memory = Memory::with_size_protect(bytes.len(), Protect::ReadWrite) + .expect("Could not create a memory"); + + unsafe { + memory.as_slice_mut().copy_from_slice(&*bytes); + + if memory.protection() != original_protection { + memory + .protect(.., original_protection) + .expect("Could not protect memory as its original protection"); + } + } + + Ok(memory) + } + } + + deserializer.deserialize_struct("Memory", &["protection", "data"], MemoryVisitor) + } +} + +/// Round `size` up to the nearest multiple of `page_size`. +fn round_up_to_page_size(size: usize, page_size: usize) -> usize { + assert!(page_size.is_power_of_two()); + (size + (page_size - 1)) & !(page_size - 1) +} + +/// Round `size` down to the nearest multiple of `page_size`. +fn round_down_to_page_size(size: usize, page_size: usize) -> usize { + assert!(page_size.is_power_of_two()); + size & !(page_size - 1) +} diff --git a/lib/runtime-core/src/sys/unix/memory.rs b/lib/runtime-core/src/sys/unix/memory.rs new file mode 100644 index 000000000000..8b56ac6b78eb --- /dev/null +++ b/lib/runtime-core/src/sys/unix/memory.rs @@ -0,0 +1,351 @@ +use crate::error::MemoryCreationError; +use crate::error::MemoryProtectionError; +use crate::sys::{round_down_to_page_size, round_up_to_page_size}; +use errno; +use nix::libc; +use page_size; +use std::ops::{Bound, RangeBounds}; +use std::{fs::File, os::unix::io::IntoRawFd, path::Path, ptr, slice, sync::Arc}; +use rkyv::{Archive, Serialize as RkyvSerialize, Deserialize as RkyvDeserialize}; + +unsafe impl Send for Memory {} +unsafe impl Sync for Memory {} + +/// Data for a sized and protected region of memory. +#[derive(Debug)] +pub struct Memory { + ptr: *mut u8, + size: usize, + protection: Protect, + fd: Option>, + content_size: u32, +} + +impl Memory { + /// Create a new memory from the given path value and protection. + pub fn from_file_path

(path: P, protection: Protect) -> Result + where + P: AsRef, + { + let file = File::open(path)?; + + let file_len = file.metadata()?.len(); + + let raw_fd = RawFd::from_file(file); + + let ptr = unsafe { + libc::mmap( + ptr::null_mut(), + file_len as usize, + protection.to_protect_const() as i32, + libc::MAP_PRIVATE, + raw_fd.0, + 0, + ) + }; + + if ptr == -1 as _ { + Err(MemoryCreationError::VirtualMemoryAllocationFailed( + file_len as usize, + errno::errno().to_string(), + )) + } else { + Ok(Self { + ptr: ptr as *mut u8, + size: file_len as usize, + protection, + fd: Some(Arc::new(raw_fd)), + content_size: 0, + }) + } + } + + /// Create a new memory with the given size and protection. + pub fn with_size_protect(size: usize, protection: Protect) -> Result { + if size == 0 { + return Ok(Self { + ptr: ptr::null_mut(), + size: 0, + protection, + fd: None, + content_size: 0, + }); + } + + let size = round_up_to_page_size(size, page_size::get()); + + let ptr = unsafe { + libc::mmap( + ptr::null_mut(), + size, + protection.to_protect_const() as i32, + libc::MAP_PRIVATE | libc::MAP_ANON, + -1, + 0, + ) + }; + + if ptr == -1 as _ { + Err(errno::errno().to_string()) + } else { + Ok(Self { + ptr: ptr as *mut u8, + size, + protection, + fd: None, + content_size: 0, + }) + } + } + + /// Create a new memory with the given contents size and protection. + /// Used when the size of the contents must be tracked (e.g. for rkyv deserialization). + pub fn with_content_size_protect(content_size: u32, protection: Protect) -> Result { + let mut memory = Self::with_size_protect(content_size as usize, protection)?; + memory.set_content_size(content_size); + Ok(memory) + } + + /// Create a new memory with the given size. + pub fn with_size(size: usize) -> Result { + if size == 0 { + return Ok(Self { + ptr: ptr::null_mut(), + size: 0, + protection: Protect::None, + fd: None, + content_size: 0, + }); + } + + let size = round_up_to_page_size(size, page_size::get()); + + let ptr = unsafe { + libc::mmap( + ptr::null_mut(), + size, + libc::PROT_NONE, + libc::MAP_PRIVATE | libc::MAP_ANON, + -1, + 0, + ) + }; + + if ptr == -1 as _ { + Err(MemoryCreationError::VirtualMemoryAllocationFailed( + size, + errno::errno().to_string(), + )) + } else { + Ok(Self { + ptr: ptr as *mut u8, + size, + protection: Protect::None, + fd: None, + content_size: 0, + }) + } + } + + /// Protect this memory with the given range bounds and protection. + pub unsafe fn protect( + &mut self, + range: impl RangeBounds, + protection: Protect, + ) -> Result<(), MemoryProtectionError> { + let protect = protection.to_protect_const(); + + let range_start = match range.start_bound() { + Bound::Included(start) => *start, + Bound::Excluded(start) => *start, + Bound::Unbounded => 0, + }; + + let range_end = match range.end_bound() { + Bound::Included(end) => *end, + Bound::Excluded(end) => *end, + Bound::Unbounded => self.size(), + }; + + let page_size = page_size::get(); + let start = self + .ptr + .add(round_down_to_page_size(range_start, page_size)); + let size = round_up_to_page_size(range_end - range_start, page_size); + assert!(size <= self.size); + + let success = libc::mprotect(start as _, size, protect as i32); + if success == -1 { + Err(MemoryProtectionError::ProtectionFailed( + start as usize, + size, + errno::errno().to_string(), + )) + } else { + self.protection = protection; + Ok(()) + } + } + + /// Set the content size of this memory. Must be set manually, as this is different in each + /// case. + pub fn set_content_size(&mut self, size: u32) { + self.content_size = size; + } + + /// Split this memory into multiple memories by the given offset. + pub fn split_at(mut self, offset: usize) -> (Memory, Memory) { + let page_size = page_size::get(); + if offset % page_size == 0 { + let second_ptr = unsafe { self.ptr.add(offset) }; + let second_size = self.size - offset; + + self.size = offset; + + let second = Memory { + ptr: second_ptr, + size: second_size, + protection: self.protection, + fd: self.fd.clone(), + content_size: 0, + }; + + (self, second) + } else { + panic!("offset must be multiple of page size: {}", offset) + } + } + + /// Gets the size of this memory. + pub fn size(&self) -> usize { + self.size + } + + /// Gets the size of the actual contents of this memory. + pub fn content_size(&self) -> u32 { + self.content_size + } + + /// Gets a slice for this memory. + pub unsafe fn as_slice(&self) -> &[u8] { + slice::from_raw_parts(self.ptr, self.size) + } + + /// Gets a slice for this memory, bounded by content_size. + pub unsafe fn as_slice_contents(&self) -> &[u8] { + slice::from_raw_parts(self.ptr, self.content_size as usize) + } + + /// Gets a mutable slice for this memory. + pub unsafe fn as_slice_mut(&mut self) -> &mut [u8] { + slice::from_raw_parts_mut(self.ptr, self.size) + } + + /// Gets the protect kind of this memory. + pub fn protection(&self) -> Protect { + self.protection + } + + /// Gets mutable pointer to the memory. + pub fn as_ptr(&self) -> *mut u8 { + self.ptr + } +} + +impl Drop for Memory { + fn drop(&mut self) { + if !self.ptr.is_null() { + let success = unsafe { libc::munmap(self.ptr as _, self.size) }; + assert_eq!(success, 0, "failed to unmap memory: {}", errno::errno()); + } + } +} + +impl Clone for Memory { + fn clone(&self) -> Self { + let temp_protection = if self.protection.is_writable() { + self.protection + } else { + Protect::ReadWrite + }; + + let mut new = Memory::with_size_protect(self.size, temp_protection).unwrap(); + unsafe { + new.as_slice_mut().copy_from_slice(self.as_slice()); + + if temp_protection != self.protection { + new.protect(.., self.protection).unwrap(); + } + } + + new + } +} + +/// Kinds of memory protection. +#[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialEq, Eq, Archive, RkyvSerialize, RkyvDeserialize)] +#[allow(dead_code)] +#[archive(compare(PartialEq))] +#[archive_attr(derive(Debug))] +#[archive_attr(derive(PartialEq))] +pub enum Protect { + /// Read/write/exec allowed. + None, + /// Read only. + Read, + /// Read/write only. + ReadWrite, + /// Read/exec only. + ReadExec, + /// Read/write/exec only. + ReadWriteExec, +} + +impl Protect { + fn to_protect_const(self) -> u32 { + match self { + Protect::None => 0, + Protect::Read => 1, + Protect::ReadWrite => 1 | 2, + Protect::ReadExec => 1 | 4, + Protect::ReadWriteExec => 1 | 2 | 4, + } + } + + /// Returns true if this memory is readable. + pub fn is_readable(self) -> bool { + match self { + Protect::Read | Protect::ReadWrite | Protect::ReadExec | Protect::ReadWriteExec => true, + _ => false, + } + } + + /// Returns true if this memory is writable. + pub fn is_writable(self) -> bool { + match self { + Protect::ReadWrite | Protect::ReadWriteExec => true, + _ => false, + } + } +} + +#[derive(Debug, Archive, RkyvSerialize, RkyvDeserialize)] +pub struct RawFd(i32); + +impl RawFd { + fn from_file(f: File) -> Self { + RawFd(f.into_raw_fd()) + } +} + +impl Drop for RawFd { + fn drop(&mut self) { + let success = unsafe { libc::close(self.0) }; + assert_eq!( + success, + 0, + "failed to close mmapped file descriptor: {}", + errno::errno() + ); + } +} diff --git a/lib/runtime-core/src/types.rs b/lib/runtime-core/src/types.rs new file mode 100644 index 000000000000..712806112cec --- /dev/null +++ b/lib/runtime-core/src/types.rs @@ -0,0 +1,607 @@ +//! The runtime types modules represent type used within the wasm runtime and helper functions to +//! convert to other represenations. + +use crate::{memory::MemoryType, module::ModuleInfo, structures::TypedIndex, units::Pages}; +use std::{borrow::Cow, convert::TryFrom}; +use rkyv::{Archive, Serialize as RkyvSerialize, Deserialize as RkyvDeserialize, with::AsOwned}; + +/// Represents a WebAssembly type. +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Hash, Archive, RkyvSerialize, RkyvDeserialize)] +pub enum Type { + /// The `i32` type. + I32, + /// The `i64` type. + I64, + /// The `f32` type. + F32, + /// The `f64` type. + F64, + /// The `v128` type. + V128, +} + +impl std::fmt::Display for Type { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + +/// Represents a WebAssembly value. +/// +/// As the number of types in WebAssembly expand, +/// this structure will expand as well. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Archive, RkyvSerialize, RkyvDeserialize)] +pub enum Value { + /// The `i32` type. + I32(i32), + /// The `i64` type. + I64(i64), + /// The `f32` type. + F32(f32), + /// The `f64` type. + F64(f64), + /// The `v128` type. + V128(u128), +} + +impl Value { + /// The `Type` of this `Value`. + pub fn ty(&self) -> Type { + match self { + Value::I32(_) => Type::I32, + Value::I64(_) => Type::I64, + Value::F32(_) => Type::F32, + Value::F64(_) => Type::F64, + Value::V128(_) => Type::V128, + } + } + + /// Convert this `Value` to a u128 binary representation. + pub fn to_u128(&self) -> u128 { + match *self { + Value::I32(x) => x as u128, + Value::I64(x) => x as u128, + Value::F32(x) => f32::to_bits(x) as u128, + Value::F64(x) => f64::to_bits(x) as u128, + Value::V128(x) => x, + } + } +} + +macro_rules! value_conversions { + ($native_type:ty, $value_variant:ident) => { + impl From<$native_type> for Value { + fn from(n: $native_type) -> Self { + Self::$value_variant(n) + } + } + + impl TryFrom<&Value> for $native_type { + type Error = &'static str; + + fn try_from(value: &Value) -> Result { + match value { + Value::$value_variant(value) => Ok(*value), + _ => Err("Invalid cast."), + } + } + } + }; +} + +value_conversions!(i32, I32); +value_conversions!(i64, I64); +value_conversions!(f32, F32); +value_conversions!(f64, F64); +value_conversions!(u128, V128); + +/// Represents a native wasm type. +pub unsafe trait NativeWasmType: Copy + Into +where + Self: Sized, +{ + /// Type for this `NativeWasmType`. + const TYPE: Type; + + /// Convert from u64 bites to self. + fn from_binary(bits: u64) -> Self; + + /// Convert self to u64 binary representation. + fn to_binary(self) -> u64; +} + +unsafe impl NativeWasmType for i32 { + const TYPE: Type = Type::I32; + + fn from_binary(bits: u64) -> Self { + bits as _ + } + + fn to_binary(self) -> u64 { + self as _ + } +} + +unsafe impl NativeWasmType for i64 { + const TYPE: Type = Type::I64; + + fn from_binary(bits: u64) -> Self { + bits as _ + } + + fn to_binary(self) -> u64 { + self as _ + } +} + +unsafe impl NativeWasmType for f32 { + const TYPE: Type = Type::F32; + + fn from_binary(bits: u64) -> Self { + f32::from_bits(bits as u32) + } + + fn to_binary(self) -> u64 { + self.to_bits() as _ + } +} + +unsafe impl NativeWasmType for f64 { + const TYPE: Type = Type::F64; + + fn from_binary(bits: u64) -> Self { + f64::from_bits(bits) + } + + fn to_binary(self) -> u64 { + self.to_bits() + } +} + +/// A trait to represent a wasm extern type. +pub unsafe trait WasmExternType: Copy +where + Self: Sized, +{ + /// Native wasm type for this `WasmExternType`. + type Native: NativeWasmType; + + /// Convert from given `Native` type to self. + fn from_native(native: Self::Native) -> Self; + + /// Convert self to `Native` type. + fn to_native(self) -> Self::Native; +} + +macro_rules! wasm_extern_type { + ($type:ty => $native_type:ty) => { + unsafe impl WasmExternType for $type { + type Native = $native_type; + + fn from_native(native: Self::Native) -> Self { + native as _ + } + + fn to_native(self) -> Self::Native { + self as _ + } + } + }; +} + +wasm_extern_type!(i8 => i32); +wasm_extern_type!(u8 => i32); +wasm_extern_type!(i16 => i32); +wasm_extern_type!(u16 => i32); +wasm_extern_type!(i32 => i32); +wasm_extern_type!(u32 => i32); +wasm_extern_type!(i64 => i64); +wasm_extern_type!(u64 => i64); +wasm_extern_type!(f32 => f32); +wasm_extern_type!(f64 => f64); + +// pub trait IntegerAtomic +// where +// Self: Sized +// { +// type Primitive; + +// fn add(&self, other: Self::Primitive) -> Self::Primitive; +// fn sub(&self, other: Self::Primitive) -> Self::Primitive; +// fn and(&self, other: Self::Primitive) -> Self::Primitive; +// fn or(&self, other: Self::Primitive) -> Self::Primitive; +// fn xor(&self, other: Self::Primitive) -> Self::Primitive; +// fn load(&self) -> Self::Primitive; +// fn store(&self, other: Self::Primitive) -> Self::Primitive; +// fn compare_exchange(&self, expected: Self::Primitive, new: Self::Primitive) -> Self::Primitive; +// fn swap(&self, other: Self::Primitive) -> Self::Primitive; +// } + +/// Trait for a Value type. A Value type is a type that is always valid and may +/// be safely copied. +/// +/// That is, for all possible bit patterns a valid Value type can be constructed +/// from those bits. +/// +/// Concretely a `u32` is a Value type because every combination of 32 bits is +/// a valid `u32`. However a `bool` is _not_ a Value type because any bit patterns +/// other than `0` and `1` are invalid in Rust and may cause undefined behavior if +/// a `bool` is constructed from those bytes. +pub unsafe trait ValueType: Copy +where + Self: Sized, +{ +} + +macro_rules! convert_value_impl { + ($t:ty) => { + unsafe impl ValueType for $t {} + }; + ( $($t:ty),* ) => { + $( + convert_value_impl!($t); + )* + }; +} + +convert_value_impl!(u8, i8, u16, i16, u32, i32, u64, i64, f32, f64); + +/// Kinds of element types. +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Archive, RkyvSerialize, RkyvDeserialize)] +pub enum ElementType { + /// Any wasm function. + Anyfunc, +} + +/// Describes the properties of a table including the element types, minimum and optional maximum, +/// number of elements in the table. +#[derive(Serialize, Deserialize, Debug, Clone, Copy, Archive, RkyvSerialize, RkyvDeserialize)] +pub struct TableDescriptor { + /// Type of data stored in this table. + pub element: ElementType, + /// The minimum number of elements that must be stored in this table. + pub minimum: u32, + /// The maximum number of elements in this table. + pub maximum: Option, +} + +impl TableDescriptor { + pub(crate) fn fits_in_imported(&self, imported: TableDescriptor) -> bool { + // TODO: We should define implementation limits. + let imported_max = imported.maximum.unwrap_or(u32::max_value()); + let self_max = self.maximum.unwrap_or(u32::max_value()); + self.element == imported.element + && imported_max <= self_max + && self.minimum <= imported.minimum + } +} + +/// A const value initializer. +/// Over time, this will be able to represent more and more +/// complex expressions. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Archive, RkyvSerialize, RkyvDeserialize)] +pub enum Initializer { + /// Corresponds to a `const.*` instruction. + Const(Value), + /// Corresponds to a `get_global` instruction. + GetGlobal(ImportedGlobalIndex), +} + +/// Describes the mutability and type of a Global +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Archive, RkyvSerialize, RkyvDeserialize)] +pub struct GlobalDescriptor { + /// Mutable flag. + pub mutable: bool, + /// Wasm type. + pub ty: Type, +} + +/// A wasm global. +#[derive(Serialize, Deserialize, Debug, Clone, Archive, RkyvSerialize, RkyvDeserialize)] +pub struct GlobalInit { + /// Global descriptor. + pub desc: GlobalDescriptor, + /// Global initializer. + pub init: Initializer, +} + +/// A wasm memory descriptor. +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Archive, RkyvSerialize, RkyvDeserialize)] +pub struct MemoryDescriptor { + /// The minimum number of allowed pages. + pub minimum: Pages, + /// The maximum number of allowed pages. + pub maximum: Option, + /// This memory can be shared between wasm threads. + pub shared: bool, + /// The type of the memory + pub memory_type: MemoryType, +} + +impl MemoryDescriptor { + /// Create a new memory descriptor with the given min/max pages and shared flag. + pub fn new(minimum: Pages, maximum: Option, shared: bool) -> Result { + let memory_type = match (maximum.is_some(), shared) { + (true, true) => MemoryType::SharedStatic, + (true, false) => MemoryType::Static, + (false, false) => MemoryType::Dynamic, + (false, true) => { + return Err("Max number of pages is required for shared memory".to_string()); + } + }; + Ok(MemoryDescriptor { + minimum, + maximum, + shared, + memory_type, + }) + } + + /// Returns the `MemoryType` for this descriptor. + pub fn memory_type(&self) -> MemoryType { + self.memory_type + } + + pub(crate) fn fits_in_imported(&self, imported: MemoryDescriptor) -> bool { + let imported_max = imported.maximum.unwrap_or(Pages(65_536)); + let self_max = self.maximum.unwrap_or(Pages(65_536)); + + self.shared == imported.shared + && imported_max <= self_max + && self.minimum <= imported.minimum + } +} + +/// The signature of a function that is either implemented +/// in a wasm module or exposed to wasm by the host. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash, Archive, RkyvSerialize, RkyvDeserialize)] +pub struct FuncSig { + #[with(AsOwned)] + params: Cow<'static, [Type]>, + #[with(AsOwned)] + returns: Cow<'static, [Type]>, +} + +impl FuncSig { + /// Creates a new function signatures with the given parameter and return types. + pub fn new(params: Params, returns: Returns) -> Self + where + Params: Into>, + Returns: Into>, + { + Self { + params: params.into(), + returns: returns.into(), + } + } + + /// Parameter types. + pub fn params(&self) -> &[Type] { + &self.params + } + + /// Return types. + pub fn returns(&self) -> &[Type] { + &self.returns + } + + /// Returns true if parameter types match the function signature. + pub fn check_param_value_types(&self, params: &[Value]) -> bool { + self.params.len() == params.len() + && self + .params + .iter() + .zip(params.iter().map(|val| val.ty())) + .all(|(t0, ref t1)| t0 == t1) + } +} + +impl std::fmt::Display for FuncSig { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let params = self + .params + .iter() + .map(|p| p.to_string()) + .collect::>() + .join(", "); + let returns = self + .returns + .iter() + .map(|p| p.to_string()) + .collect::>() + .join(", "); + write!(f, "[{}] -> [{}]", params, returns) + } +} + +/// Trait that represents Local or Import. +pub trait LocalImport { + /// Local type. + type Local: TypedIndex; + /// Import type. + type Import: TypedIndex; +} + +#[rustfmt::skip] +macro_rules! define_map_index { + ($ty:ident) => { + /// Typed Index + #[derive(Serialize, Deserialize, Archive, RkyvSerialize, RkyvDeserialize)] + #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] + pub struct $ty (u32); + impl TypedIndex for $ty { + #[doc(hidden)] + fn new(index: usize) -> Self { + $ty (index as _) + } + + #[doc(hidden)] + fn index(&self) -> usize { + self.0 as usize + } + } + }; + ($($normal_ty:ident,)* | local: $($local_ty:ident,)* | imported: $($imported_ty:ident,)*) => { + $( + define_map_index!($normal_ty); + define_map_index!($local_ty); + define_map_index!($imported_ty); + + impl LocalImport for $normal_ty { + type Local = $local_ty; + type Import = $imported_ty; + } + )* + }; +} + +#[rustfmt::skip] +define_map_index![ + FuncIndex, MemoryIndex, TableIndex, GlobalIndex, + | local: LocalFuncIndex, LocalMemoryIndex, LocalTableIndex, LocalGlobalIndex, + | imported: ImportedFuncIndex, ImportedMemoryIndex, ImportedTableIndex, ImportedGlobalIndex, +]; + +#[rustfmt::skip] +macro_rules! define_local_or_import { + ($ty:ident, $local_ty:ident, $imported_ty:ident, $imports:ident) => { + impl $ty { + /// Converts self into `LocalOrImport`. + pub fn local_or_import(self, info: &ModuleInfo) -> LocalOrImport<$ty> { + if self.index() < info.$imports.len() { + LocalOrImport::Import(::Import::new(self.index())) + } else { + LocalOrImport::Local(::Local::new(self.index() - info.$imports.len())) + } + } + } + + impl $local_ty { + /// Convert up. + pub fn convert_up(self, info: &ModuleInfo) -> $ty { + $ty ((self.index() + info.$imports.len()) as u32) + } + } + + impl $imported_ty { + /// Convert up. + pub fn convert_up(self, _info: &ModuleInfo) -> $ty { + $ty (self.index() as u32) + } + } + }; + ($(($ty:ident | ($local_ty:ident, $imported_ty:ident): $imports:ident),)*) => { + $( + define_local_or_import!($ty, $local_ty, $imported_ty, $imports); + )* + }; +} + +#[rustfmt::skip] +define_local_or_import![ + (FuncIndex | (LocalFuncIndex, ImportedFuncIndex): imported_functions), + (MemoryIndex | (LocalMemoryIndex, ImportedMemoryIndex): imported_memories), + (TableIndex | (LocalTableIndex, ImportedTableIndex): imported_tables), + (GlobalIndex | (LocalGlobalIndex, ImportedGlobalIndex): imported_globals), +]; + +/// Index for signature. +#[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialEq, Eq, Hash, Archive, RkyvSerialize, RkyvDeserialize)] +pub struct SigIndex(u32); +impl TypedIndex for SigIndex { + #[doc(hidden)] + fn new(index: usize) -> Self { + SigIndex(index as _) + } + + #[doc(hidden)] + fn index(&self) -> usize { + self.0 as usize + } +} + +/// Kind of local or import type. +pub enum LocalOrImport +where + T: LocalImport, +{ + /// Local. + Local(T::Local), + /// Import. + Import(T::Import), +} + +impl LocalOrImport +where + T: LocalImport, +{ + /// Returns `Some` if self is local, `None` if self is an import. + pub fn local(self) -> Option { + match self { + LocalOrImport::Local(local) => Some(local), + LocalOrImport::Import(_) => None, + } + } + + /// Returns `Some` if self is an import, `None` if self is local. + pub fn import(self) -> Option { + match self { + LocalOrImport::Import(import) => Some(import), + LocalOrImport::Local(_) => None, + } + } +} + +#[cfg(test)] +mod tests { + use crate::types::NativeWasmType; + use crate::types::WasmExternType; + + #[test] + fn test_native_types_round_trip() { + assert_eq!( + 42i32, + i32::from_native(i32::from_binary((42i32).to_native().to_binary())) + ); + + assert_eq!( + -42i32, + i32::from_native(i32::from_binary((-42i32).to_native().to_binary())) + ); + + use std::i64; + let xi64 = i64::MAX; + assert_eq!( + xi64, + i64::from_native(i64::from_binary((xi64).to_native().to_binary())) + ); + let yi64 = i64::MIN; + assert_eq!( + yi64, + i64::from_native(i64::from_binary((yi64).to_native().to_binary())) + ); + + assert_eq!( + 16.5f32, + f32::from_native(f32::from_binary((16.5f32).to_native().to_binary())) + ); + + assert_eq!( + -16.5f32, + f32::from_native(f32::from_binary((-16.5f32).to_native().to_binary())) + ); + + use std::f64; + let xf64: f64 = f64::MAX; + assert_eq!( + xf64, + f64::from_native(f64::from_binary((xf64).to_native().to_binary())) + ); + + let yf64: f64 = f64::MIN; + assert_eq!( + yf64, + f64::from_native(f64::from_binary((yf64).to_native().to_binary())) + ); + } +} diff --git a/lib/runtime-core/src/units.rs b/lib/runtime-core/src/units.rs new file mode 100644 index 000000000000..0df5bdec2c07 --- /dev/null +++ b/lib/runtime-core/src/units.rs @@ -0,0 +1,110 @@ +//! The units module provides common WebAssembly units like [`Pages`] and conversion functions into +//! other units. +use crate::error::PageError; +use std::{ + fmt, + ops::{Add, Sub}, +}; + +use rkyv::{Archive, Serialize as RkyvSerialize, Deserialize as RkyvDeserialize}; + +/// The page size in bytes of a wasm page. +pub const WASM_PAGE_SIZE: usize = 65_536; +/// The max number of wasm pages allowed. +pub const WASM_MAX_PAGES: usize = 65_536; +// From emscripten resize_heap implementation +/// The minimum number of wasm pages allowed. +pub const WASM_MIN_PAGES: usize = 256; + +/// Units of WebAssembly pages (as specified to be 65,536 bytes). +#[derive(Serialize, Deserialize, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Archive, RkyvSerialize, RkyvDeserialize)] +pub struct Pages(pub u32); + +impl Pages { + /// Checked add of Pages to Pages. + pub fn checked_add(self, rhs: Pages) -> Result { + let added = (self.0 as usize) + (rhs.0 as usize); + if added <= WASM_MAX_PAGES { + Ok(Pages(added as u32)) + } else { + Err(PageError::ExceededMaxPages( + self.0 as usize, + rhs.0 as usize, + added, + )) + } + } + + /// Calculate number of bytes from pages. + pub fn bytes(self) -> Bytes { + self.into() + } +} + +impl fmt::Debug for Pages { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{} pages", self.0) + } +} + +/// Units of WebAssembly memory in terms of 8-bit bytes. +#[derive(Serialize, Deserialize, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct Bytes(pub usize); + +impl fmt::Debug for Bytes { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{} bytes", self.0) + } +} + +impl From for Bytes { + fn from(pages: Pages) -> Bytes { + Bytes((pages.0 as usize) * WASM_PAGE_SIZE) + } +} + +impl Sub for Pages +where + T: Into, +{ + type Output = Pages; + fn sub(self, rhs: T) -> Pages { + Pages(self.0 - rhs.into().0) + } +} + +impl Add for Pages +where + T: Into, +{ + type Output = Pages; + fn add(self, rhs: T) -> Pages { + Pages(self.0 + rhs.into().0) + } +} + +impl From for Pages { + fn from(bytes: Bytes) -> Pages { + Pages((bytes.0 / WASM_PAGE_SIZE) as u32) + } +} + +impl Sub for Bytes +where + T: Into, +{ + type Output = Bytes; + fn sub(self, rhs: T) -> Bytes { + Bytes(self.0 - rhs.into().0) + } +} + +impl Add for Bytes +where + T: Into, +{ + type Output = Bytes; + fn add(self, rhs: T) -> Bytes { + Bytes(self.0 + rhs.into().0) + } +} diff --git a/lib/runtime/Cargo.toml b/lib/runtime/Cargo.toml new file mode 100644 index 000000000000..b33c5ab58d7f --- /dev/null +++ b/lib/runtime/Cargo.toml @@ -0,0 +1,69 @@ +[package] +name = "wasmer-runtime" +version = "0.15.0" +description = "Wasmer runtime library" +license = "MIT" +authors = ["The Wasmer Engineering Team "] +repository = "https://github.com/ElrondNetwork/wasmer" +keywords = ["wasm", "webassembly", "runtime", "sandbox", "secure"] +categories = ["wasm", "api-bindings"] +edition = "2018" +readme = "README.md" + +[dependencies] +lazy_static = "1.4" +memmap = "0.7" + +[dependencies.wasmer-runtime-core] +path = "../runtime-core" +version = "0.15.0" + +[dependencies.wasmer-clif-backend] +path = "../clif-backend" +version = "0.15.0" +optional = true + +[dependencies.wasmer-llvm-backend] +path = "../llvm-backend" +version = "0.15.0" +optional = true + +[dependencies.wasmer-singlepass-backend] +path = "../singlepass-backend" +version = "0.15.0" +optional = true + +# Dependencies for caching. +[dependencies.serde] +version = "1.0" +# This feature is required for serde to support serializing/deserializing reference counted pointers (e.g. Rc and Arc). +features = ["rc"] +[dependencies.serde_derive] +version = "1.0" + +[dev-dependencies] +tempfile = "3.1" +criterion = "0.2" +wabt = "0.9.1" + + +[features] +default = ["singlepass", "default-backend-singlepass"] +docs = [] +cranelift = ["wasmer-clif-backend"] +cache = ["cranelift"] +debug = ["wasmer-runtime-core/debug"] +llvm = ["wasmer-llvm-backend"] +singlepass = ["wasmer-singlepass-backend"] +default-backend-cranelift = ["cranelift"] +default-backend-singlepass = ["singlepass"] +default-backend-llvm = ["llvm"] +deterministic-execution = ["wasmer-singlepass-backend/deterministic-execution", "wasmer-runtime-core/deterministic-execution"] + +[[bench]] +name = "nginx" +harness = false + +[[bench]] +name = "many_instances" +harness = false diff --git a/lib/runtime/src/lib.rs b/lib/runtime/src/lib.rs new file mode 100644 index 000000000000..d9166a0a5dd6 --- /dev/null +++ b/lib/runtime/src/lib.rs @@ -0,0 +1,392 @@ +#![deny( + dead_code, + missing_docs, + nonstandard_style, + unused_imports, + unused_mut, + unused_variables, + unused_unsafe, + unreachable_patterns +)] +#![doc(html_favicon_url = "https://wasmer.io/static/icons/favicon.ico")] +#![doc(html_logo_url = "https://avatars3.githubusercontent.com/u/44205449?s=200&v=4")] + +//! Wasmer-runtime is a library that makes embedding WebAssembly +//! in your application easy, efficient, and safe. +//! +//! # How to use Wasmer-Runtime +//! +//! The easiest way is to use the [`instantiate`] function to create an [`Instance`]. +//! Then you can use [`call`] or [`func`] and then [`call`][func.call] to call an exported function safely. +//! +//! [`instantiate`]: fn.instantiate.html +//! [`Instance`]: struct.Instance.html +//! [`call`]: struct.Instance.html#method.call +//! [`func`]: struct.Instance.html#method.func +//! [func.call]: struct.Function.html#method.call +//! +//! ## Here's an example: +//! +//! Given this WebAssembly: +//! +//! ```wat +//! (module +//! (type $t0 (func (param i32) (result i32))) +//! (func $add_one (export "add_one") (type $t0) (param $p0 i32) (result i32) +//! get_local $p0 +//! i32.const 1 +//! i32.add)) +//! ``` +//! +//! compiled into wasm bytecode, we can call the exported "add_one" function: +//! +//! ``` +//! static WASM: &'static [u8] = &[ +//! // The module above compiled to bytecode goes here. +//! // ... +//! # 0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x01, 0x06, 0x01, 0x60, +//! # 0x01, 0x7f, 0x01, 0x7f, 0x03, 0x02, 0x01, 0x00, 0x07, 0x0b, 0x01, 0x07, +//! # 0x61, 0x64, 0x64, 0x5f, 0x6f, 0x6e, 0x65, 0x00, 0x00, 0x0a, 0x09, 0x01, +//! # 0x07, 0x00, 0x20, 0x00, 0x41, 0x01, 0x6a, 0x0b, 0x00, 0x1a, 0x04, 0x6e, +//! # 0x61, 0x6d, 0x65, 0x01, 0x0a, 0x01, 0x00, 0x07, 0x61, 0x64, 0x64, 0x5f, +//! # 0x6f, 0x6e, 0x65, 0x02, 0x07, 0x01, 0x00, 0x01, 0x00, 0x02, 0x70, 0x30, +//! ]; +//! +//! use wasmer_runtime::{ +//! instantiate, +//! Value, +//! imports, +//! error, +//! Func, +//! }; +//! +//! fn main() -> error::Result<()> { +//! // We're not importing anything, so make an empty import object. +//! let import_object = imports! {}; +//! +//! let mut instance = instantiate(WASM, &import_object)?; +//! +//! let add_one: Func = instance.func("add_one")?; +//! +//! let value = add_one.call(42)?; +//! +//! assert_eq!(value, 43); +//! +//! Ok(()) +//! } +//! ``` +//! +//! # Additional Notes: +//! +//! `wasmer-runtime` is built to support multiple compiler backends. +//! Currently, we support the Singlepass, [Cranelift], and LLVM compilers +//! with the [`wasmer-singlepass-backend`], [`wasmer-clif-backend`], and +//! wasmer-llvm-backend crates, respectively. +//! +//! You can specify the compiler you wish to use with the [`compile_with`] +//! function or use the default with the [`compile`] function. +//! +//! [Cranelift]: https://github.com/CraneStation/cranelift +//! [LLVM]: https://llvm.org +//! [`wasmer-singlepass-backend`]: https://crates.io/crates/wasmer-singlepass-backend +//! [`wasmer-clif-backend`]: https://crates.io/crates/wasmer-clif-backend + +#[macro_use] +extern crate serde_derive; + +pub use wasmer_runtime_core::backend::{ExceptionCode, Features}; +pub use wasmer_runtime_core::codegen::{MiddlewareChain, StreamingCompiler}; +pub use wasmer_runtime_core::export::Export; +pub use wasmer_runtime_core::global::Global; +pub use wasmer_runtime_core::import::{ImportObject, LikeNamespace}; +pub use wasmer_runtime_core::instance::{DynFunc, Instance}; +pub use wasmer_runtime_core::memory::ptr::{Array, Item, WasmPtr}; +pub use wasmer_runtime_core::memory::Memory; +pub use wasmer_runtime_core::module::Module; +pub use wasmer_runtime_core::table::Table; +pub use wasmer_runtime_core::types::Value; +pub use wasmer_runtime_core::vm::Ctx; + +pub use wasmer_runtime_core::Func; +pub use wasmer_runtime_core::{compile_with, validate}; +pub use wasmer_runtime_core::{func, imports}; + +#[cfg(unix)] +pub use wasmer_runtime_core::{ + fault::{pop_code_version, push_code_version}, + state::CodeVersion, +}; + +pub mod memory { + //! The memory module contains the implementation data structures and helper functions used to + //! manipulate and access wasm memory. + pub use wasmer_runtime_core::memory::{Atomically, Memory, MemoryView}; +} + +pub mod wasm { + //! Various types exposed by the Wasmer Runtime. + pub use wasmer_runtime_core::global::Global; + pub use wasmer_runtime_core::table::Table; + pub use wasmer_runtime_core::types::{ + FuncSig, GlobalDescriptor, MemoryDescriptor, TableDescriptor, Type, Value, + }; +} + +pub mod error { + //! The error module contains the data structures and helper functions used to implement errors that + //! are produced and returned from the wasmer runtime. + pub use wasmer_runtime_core::cache::Error as CacheError; + pub use wasmer_runtime_core::error::*; +} + +pub mod units { + //! Various unit types. + pub use wasmer_runtime_core::units::{Bytes, Pages}; +} + +pub mod types { + //! Types used in the Wasm runtime and conversion functions. + pub use wasmer_runtime_core::types::*; +} + +pub mod cache; + +pub use wasmer_runtime_core::backend::{Compiler, CompilerConfig}; + +/// Enum used to select which compiler should be used to generate code. +#[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialEq, Eq)] +pub enum Backend { + #[cfg(feature = "singlepass")] + /// Singlepass backend + Singlepass, + #[cfg(feature = "cranelift")] + /// Cranelift backend + Cranelift, + #[cfg(feature = "llvm")] + /// LLVM backend + LLVM, + /// Auto backend + Auto, +} + +impl Backend { + /// Get a list of the currently enabled (via feature flag) backends. + pub fn variants() -> &'static [&'static str] { + &[ + #[cfg(feature = "singlepass")] + "singlepass", + #[cfg(feature = "cranelift")] + "cranelift", + #[cfg(feature = "llvm")] + "llvm", + "auto", + ] + } + + /// Stable string representation of the backend. + /// It can be used as part of a cache key, for example. + pub fn to_string(&self) -> &'static str { + match self { + #[cfg(feature = "singlepass")] + Backend::Singlepass => "singlepass", + #[cfg(feature = "cranelift")] + Backend::Cranelift => "cranelift", + #[cfg(feature = "llvm")] + Backend::LLVM => "llvm", + Backend::Auto => "auto", + } + } +} + +impl Default for Backend { + fn default() -> Self { + #[cfg(all(feature = "default-backend-singlepass", not(feature = "docs")))] + return Backend::Singlepass; + + #[cfg(any(feature = "default-backend-cranelift", feature = "docs"))] + return Backend::Cranelift; + + #[cfg(all(feature = "default-backend-llvm", not(feature = "docs")))] + return Backend::LLVM; + } +} + +impl std::str::FromStr for Backend { + type Err = String; + fn from_str(s: &str) -> Result { + match s.to_lowercase().as_str() { + #[cfg(feature = "singlepass")] + "singlepass" => Ok(Backend::Singlepass), + #[cfg(feature = "cranelift")] + "cranelift" => Ok(Backend::Cranelift), + #[cfg(feature = "llvm")] + "llvm" => Ok(Backend::LLVM), + "auto" => Ok(Backend::Auto), + _ => Err(format!("The backend {} doesn't exist", s)), + } + } +} + +/// Compile WebAssembly binary code into a [`Module`]. +/// This function is useful if it is necessary to +/// compile a module before it can be instantiated +/// (otherwise, the [`instantiate`] function should be used). +/// +/// [`Module`]: struct.Module.html +/// [`instantiate`]: fn.instantiate.html +/// +/// # Params: +/// * `wasm`: A `&[u8]` containing the +/// binary code of the wasm module you want to compile. +/// # Errors: +/// If the operation fails, the function returns `Err(error::CompileError::...)`. +pub fn compile(wasm: &[u8]) -> error::CompileResult { + wasmer_runtime_core::compile_with(&wasm[..], &default_compiler()) +} + +/// The same as `compile` but takes a `CompilerConfig` for the purpose of +/// changing the compiler's behavior +pub fn compile_with_config( + wasm: &[u8], + compiler_config: CompilerConfig, +) -> error::CompileResult { + wasmer_runtime_core::compile_with_config(&wasm[..], &default_compiler(), compiler_config) +} + +/// The same as `compile_with_config` but takes a `Compiler` for the purpose of +/// changing the backend. +pub fn compile_with_config_with( + wasm: &[u8], + compiler_config: CompilerConfig, + compiler: &dyn Compiler, +) -> error::CompileResult { + wasmer_runtime_core::compile_with_config(&wasm[..], compiler, compiler_config) +} + +/// Compile and instantiate WebAssembly code without +/// creating a [`Module`]. +/// +/// [`Module`]: struct.Module.html +/// +/// # Params: +/// * `wasm`: A `&[u8]` containing the +/// binary code of the wasm module you want to compile. +/// * `import_object`: An object containing the values to be imported +/// into the newly-created Instance, such as functions or +/// Memory objects. There must be one matching property +/// for each declared import of the compiled module or else a +/// LinkError is thrown. +/// # Errors: +/// If the operation fails, the function returns a +/// `error::CompileError`, `error::LinkError`, or +/// `error::RuntimeError` (all combined into an `error::Error`), +/// depending on the cause of the failure. +pub fn instantiate(wasm: &[u8], import_object: &ImportObject) -> error::Result { + let module = compile(wasm)?; + module.instantiate(import_object) +} + +/// Get a single instance of the default compiler to use. +/// +/// The output of this function can be controlled by the mutually +/// exclusive `default-backend-llvm`, `default-backend-singlepass`, +/// and `default-backend-cranelift` feature flags. +pub fn default_compiler() -> impl Compiler { + #[cfg(any( + all( + feature = "default-backend-llvm", + not(feature = "docs"), + any( + feature = "default-backend-cranelift", + feature = "default-backend-singlepass", + ) + ), + all( + not(feature = "docs"), + feature = "default-backend-cranelift", + any( + feature = "default-backend-singlepass", + feature = "default-backend-llvm", + ) + ), + all( + feature = "default-backend-singlepass", + not(feature = "docs"), + any( + feature = "default-backend-cranelift", + feature = "default-backend-llvm", + ) + ) + ))] + compile_error!( + "The `default-backend-X` features are mutually exclusive. Please choose just one" + ); + + #[cfg(all(feature = "default-backend-llvm", not(feature = "docs")))] + use wasmer_llvm_backend::LLVMCompiler as DefaultCompiler; + + #[cfg(any( + feature = "default-backend-singlepass", + feature = "deterministic-execution" + ))] + use wasmer_singlepass_backend::SinglePassCompiler as DefaultCompiler; + + #[cfg(any(feature = "default-backend-cranelift", feature = "docs"))] + use wasmer_clif_backend::CraneliftCompiler as DefaultCompiler; + + DefaultCompiler::new() +} + +/// Get the `Compiler` as a trait object for the given `Backend`. +/// Returns `Option` because support for the requested `Compiler` may +/// not be enabled by feature flags. +/// +/// To get a list of the enabled backends as strings, call `Backend::variants()`. +pub fn compiler_for_backend(backend: Backend) -> Option> { + match backend { + #[cfg(feature = "cranelift")] + Backend::Cranelift => Some(Box::new(wasmer_clif_backend::CraneliftCompiler::new())), + + #[cfg(any(feature = "singlepass"))] + Backend::Singlepass => Some(Box::new( + wasmer_singlepass_backend::SinglePassCompiler::new(), + )), + + #[cfg(feature = "llvm")] + Backend::LLVM => Some(Box::new(wasmer_llvm_backend::LLVMCompiler::new())), + + #[cfg(any( + not(feature = "llvm"), + not(feature = "singlepass"), + not(feature = "cranelift"), + not(feature = "deterministic-execution"), + ))] + _ => None, + } +} + +/// The current version of this crate. +pub const VERSION: &str = env!("CARGO_PKG_VERSION"); + +#[cfg(test)] +mod test { + use super::*; + use std::str::FromStr; + + #[test] + fn str_repr_matches() { + // if this test breaks, think hard about why it's breaking + // can we avoid having these be different? + + for &backend in &[ + #[cfg(feature = "cranelift")] + Backend::Cranelift, + #[cfg(feature = "llvm")] + Backend::LLVM, + #[cfg(feature = "singlepass")] + Backend::Singlepass, + ] { + assert_eq!(backend, Backend::from_str(backend.to_string()).unwrap()); + } + } +} diff --git a/lib/singlepass-backend/Cargo.toml b/lib/singlepass-backend/Cargo.toml new file mode 100644 index 000000000000..dc5ee96aecb2 --- /dev/null +++ b/lib/singlepass-backend/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "wasmer-singlepass-backend" +repository = "https://github.com/ElrondNetwork/wasmer" +version = "0.15.0" +description = "Wasmer runtime single pass compiler backend" +license = "MIT" +authors = ["The Wasmer Engineering Team "] +keywords = ["wasm", "webassembly", "compiler", "JIT", "AOT"] +categories = ["wasm"] +edition = "2018" +readme = "README.md" + +[dependencies] +wasmer-runtime-core = { path = "../runtime-core", version = "0.15.0" } +dynasm = "0.7" +dynasmrt = "0.7" +lazy_static = "1.4" +byteorder = "1.3" +nix = "0.15" +libc = "0.2.60" +smallvec = "0.6" +wasmparser = { git = "https://github.com/ElrondNetwork/wasmparser.rs" } +bincode = "1.2" +serde = "1.0" +serde_derive = "1.0" + +[dependencies.rkyv] +version = "0.7.26" + +[features] +default = ["deterministic-execution"] +deterministic-execution = ["wasmparser/deterministic", "wasmer-runtime-core/deterministic-execution"] diff --git a/lib/singlepass-backend/src/codegen_x64.rs b/lib/singlepass-backend/src/codegen_x64.rs new file mode 100644 index 000000000000..116e0deb0ee1 --- /dev/null +++ b/lib/singlepass-backend/src/codegen_x64.rs @@ -0,0 +1,10020 @@ +#![allow(clippy::forget_copy)] // Used by dynasm. +#![warn(unused_imports)] + +use crate::emitter_x64::*; +use crate::machine::*; +#[cfg(target_arch = "aarch64")] +use dynasmrt::aarch64::Assembler; +#[cfg(target_arch = "x86_64")] +use dynasmrt::x64::Assembler; +use dynasmrt::{AssemblyOffset, DynamicLabel, DynasmApi, DynasmLabelApi}; +use smallvec::SmallVec; +use std::{ + any::Any, + collections::{BTreeMap, HashMap}, + ffi::c_void, + iter, mem, + ptr::NonNull, + slice, + sync::{Arc, RwLock}, + usize, + convert::TryInto, +}; + +use rkyv::{ + Archive, + Archived, + Serialize as RkyvSerialize, + Deserialize as RkyvDeserialize, + ser::Serializer, + ser::serializers::AllocSerializer, +}; + +use bincode; + +use wasmer_runtime_core::{ + backend::{ + sys::{Memory, Protect}, + Architecture, CacheGen, CompilerConfig, ExceptionCode, ExceptionTable, InlineBreakpoint, + InlineBreakpointType, MemoryBoundCheckMode, RunnableModule, Token, + }, + cache::{Artifact, Error as CacheError}, + codegen::*, + fault::{self, raw::register_preservation_trampoline}, + loader::CodeMemory, + memory::MemoryType, + module::{ModuleInfo, ModuleInner}, + state::{ + x64::new_machine_state, x64::X64Register, FunctionStateMap, MachineState, MachineValue, + ModuleStateMap, OffsetInfo, SuspendOffset, WasmAbstractValue, + }, + structures::{Map, TypedIndex}, + typed_func::{Trampoline, Wasm}, + types::{ + FuncIndex, FuncSig, GlobalIndex, LocalFuncIndex, LocalOrImport, MemoryIndex, SigIndex, + TableIndex, Type, + }, + vm::{self, LocalGlobal, LocalTable, INTERNALS_SIZE}, + wasmparser::{MemoryImmediate, Operator, Type as WpType, TypeOrFuncType as WpTypeOrFuncType}, +}; + +#[cfg(target_arch = "aarch64")] +#[allow(dead_code)] +static ARCH: Architecture = Architecture::Aarch64; +#[cfg(target_arch = "x86_64")] +#[allow(dead_code)] +static ARCH: Architecture = Architecture::X64; + +/// Inline breakpoint size for x86-64. +pub const INLINE_BREAKPOINT_SIZE_X86_SINGLEPASS: usize = 7; + +/// Inline breakpoint size for aarch64. +pub const INLINE_BREAKPOINT_SIZE_AARCH64_SINGLEPASS: usize = 12; + +pub static mut USE_RKYV_SERIALIZATION: bool = false; + +static BACKEND_ID: &str = "singlepass"; + +#[cfg(target_arch = "x86_64")] +lazy_static! { + /// Performs a System V call to `target` with [stack_top..stack_base] as the argument list, from right to left. + static ref CONSTRUCT_STACK_AND_CALL_WASM: unsafe extern "C" fn (stack_top: *const u64, stack_base: *const u64, ctx: *mut vm::Ctx, target: *const vm::Func) -> u64 = { + let mut assembler = Assembler::new().unwrap(); + let offset = assembler.offset(); + dynasm!( + assembler + ; push r15 + ; push r14 + ; push r13 + ; push r12 + ; push r11 + ; push rbp + ; mov rbp, rsp + + ; mov r15, rdi + ; mov r14, rsi + ; mov r13, rdx + ; mov r12, rcx + + ; mov rdi, r13 // ctx + + ; sub r14, 8 + ; cmp r14, r15 + ; jb >stack_ready + + ; mov rsi, [r14] + ; sub r14, 8 + ; cmp r14, r15 + ; jb >stack_ready + + ; mov rdx, [r14] + ; sub r14, 8 + ; cmp r14, r15 + ; jb >stack_ready + + ; mov rcx, [r14] + ; sub r14, 8 + ; cmp r14, r15 + ; jb >stack_ready + + ; mov r8, [r14] + ; sub r14, 8 + ; cmp r14, r15 + ; jb >stack_ready + + ; mov r9, [r14] + ; sub r14, 8 + ; cmp r14, r15 + ; jb >stack_ready + + ; mov rax, r14 + ; sub rax, r15 + ; sub rsp, rax + ; sub rsp, 8 + ; mov rax, QWORD 0xfffffffffffffff0u64 as i64 + ; and rsp, rax + ; mov rax, rsp + ; loop_begin: + ; mov r11, [r14] + ; mov [rax], r11 + ; sub r14, 8 + ; add rax, 8 + ; cmp r14, r15 + ; jb >stack_ready + ; jmp u64, userdata: *mut u8) -> u64 = { + let mut assembler = Assembler::new().unwrap(); + let offset = assembler.offset(); + dynasm!( + assembler + ; .arch aarch64 + ; sub x0, x0, 16 + ; mov x8, sp + ; str x8, [x0, 0] + ; str x30, [x0, 8] + ; adr x30, >done + ; mov sp, x0 + ; mov x0, x2 + ; br x1 + ; done: + ; ldr x30, [sp, 8] + ; ldr x8, [sp, 0] + ; mov sp, x8 + ; br x30 + ); + let buf = assembler.finalize().unwrap(); + let ret = unsafe { mem::transmute(buf.ptr(offset)) }; + mem::forget(buf); + ret + }; +} + +pub struct X64ModuleCodeGenerator { + functions: Vec, + signatures: Option>>, + function_signatures: Option>>, + function_labels: Option)>>, + assembler: Option, + func_import_count: usize, + + config: Option>, +} + +pub struct X64FunctionCode { + local_function_id: usize, + + signatures: Arc>, + function_signatures: Arc>, + fsm: FunctionStateMap, + offset: usize, + + assembler: Option, + function_labels: Option)>>, + breakpoints: Option< + HashMap< + AssemblyOffset, + Box Result<(), Box> + Send + Sync + 'static>, + >, + >, + returns: SmallVec<[WpType; 1]>, + locals: Vec, + num_params: usize, + num_locals: usize, + value_stack: Vec, + control_stack: Vec, + machine: Machine, + unreachable_depth: usize, + + config: Arc, + + exception_table: Option, +} + +enum FuncPtrInner {} +#[repr(transparent)] +#[derive(Copy, Clone, Debug)] +struct FuncPtr(*const FuncPtrInner); +unsafe impl Send for FuncPtr {} +unsafe impl Sync for FuncPtr {} + +pub struct X64ExecutionContext { + #[allow(dead_code)] + code: CodeMemory, + function_pointers: Vec, + function_offsets: Vec, + signatures: Arc>, + breakpoints: BreakpointMap, + func_import_count: usize, + msm: ModuleStateMap, + exception_table: Option, +} + +/// On-disk cache format. +/// Offsets are relative to the start of the executable image. +#[derive(Clone, Debug, Serialize, Deserialize, Archive, RkyvSerialize, RkyvDeserialize)] +pub struct CacheImage { + /// The executable image. + code: Vec, + + /// Offsets to the start of each function. Including trampoline, if any. + /// Trampolines are only present on AArch64. + /// On x86-64, `function_pointers` are identical to `function_offsets`. + function_pointers: Vec, + + /// Offsets to the start of each function after trampoline. + function_offsets: Vec, + + /// Number of imported functions. + func_import_count: usize, + + /// Module state map. + msm: ModuleStateMap, + + /// An exception table that maps instruction offsets to exception codes. + exception_table: Option, +} + +#[derive(Debug)] +pub struct ControlFrame { + pub label: DynamicLabel, + pub loop_like: bool, + pub if_else: IfElseState, + pub returns: SmallVec<[WpType; 1]>, + pub value_stack_depth: usize, + pub state: MachineState, + pub state_diff_id: usize, +} + +#[derive(Debug, Copy, Clone)] +pub enum IfElseState { + None, + If(DynamicLabel), + Else, +} + +pub struct SinglepassCache { + buffer: Arc<[u8]>, +} + +impl CacheGen for SinglepassCache { + fn generate_cache(&self) -> Result<(Box<[u8]>, Memory), CacheError> { + let content_size: u32 = self.buffer.len().try_into().unwrap(); + let mut memory = Memory::with_content_size_protect(content_size, Protect::ReadWrite) + .map_err(CacheError::SerializeError)?; + + let buffer = &*self.buffer; + + unsafe { + memory.as_slice_mut()[..buffer.len()].copy_from_slice(buffer); + } + + Ok(([].as_ref().into(), memory)) + } +} + +impl RunnableModule for X64ExecutionContext { + fn get_func( + &self, + _: &ModuleInfo, + local_func_index: LocalFuncIndex, + ) -> Option> { + self.function_pointers[self.func_import_count..] + .get(local_func_index.index()) + .and_then(|ptr| NonNull::new(ptr.0 as *mut vm::Func)) + } + + fn get_module_state_map(&self) -> Option { + Some(self.msm.clone()) + } + + fn get_breakpoints(&self) -> Option { + Some(self.breakpoints.clone()) + } + + fn get_exception_table(&self) -> Option<&ExceptionTable> { + match &self.exception_table { + Some(etable) => { Some(&etable) } + None => None + } + } + + unsafe fn patch_local_function(&self, idx: usize, target_address: usize) -> bool { + /* + 0: 48 b8 42 42 42 42 42 42 42 42 movabsq $4774451407313060418, %rax + a: 49 bb 43 43 43 43 43 43 43 43 movabsq $4846791580151137091, %r11 + 14: 41 ff e3 jmpq *%r11 + */ + #[repr(packed)] + struct LocalTrampoline { + movabsq_rax: [u8; 2], + addr_rax: u64, + movabsq_r11: [u8; 2], + addr_r11: u64, + jmpq_r11: [u8; 3], + } + + self.code.make_writable(); + + let trampoline = &mut *(self.function_pointers[self.func_import_count + idx].0 + as *const LocalTrampoline as *mut LocalTrampoline); + trampoline.movabsq_rax[0] = 0x48; + trampoline.movabsq_rax[1] = 0xb8; + trampoline.addr_rax = target_address as u64; + trampoline.movabsq_r11[0] = 0x49; + trampoline.movabsq_r11[1] = 0xbb; + trampoline.addr_r11 = + register_preservation_trampoline as unsafe extern "C" fn() as usize as u64; + trampoline.jmpq_r11[0] = 0x41; + trampoline.jmpq_r11[1] = 0xff; + trampoline.jmpq_r11[2] = 0xe3; + + self.code.make_executable(); + true + } + + fn get_trampoline(&self, _: &ModuleInfo, sig_index: SigIndex) -> Option { + // Correctly unwinding from `catch_unsafe_unwind` on hardware exceptions depends + // on the signal handlers being installed. Here we call `ensure_sighandler` "statically" + // outside `invoke()`. + fault::ensure_sighandler(); + + unsafe extern "C" fn invoke( + _trampoline: Trampoline, + ctx: *mut vm::Ctx, + func: NonNull, + args: *const u64, + rets: *mut u64, + error_out: *mut Option>, + num_params_plus_one: Option>, + ) -> bool { + let rm: &Box = &(&*(*ctx).module).runnable_module; + + let args = + slice::from_raw_parts(args, num_params_plus_one.unwrap().as_ptr() as usize - 1); + + let ret = match fault::catch_unsafe_unwind( + || { + // Puts the arguments onto the stack and calls Wasm entry. + #[cfg(target_arch = "x86_64")] + { + let args_reverse: SmallVec<[u64; 8]> = args.iter().cloned().rev().collect(); + CONSTRUCT_STACK_AND_CALL_WASM( + args_reverse.as_ptr(), + args_reverse.as_ptr().offset(args_reverse.len() as isize), + ctx, + func.as_ptr(), + ) + } + + // FIXME: Currently we are doing a hack here to convert between native aarch64 and + // "emulated" x86 ABIs. Ideally, this should be done using handwritten assembly. + #[cfg(target_arch = "aarch64")] + { + struct CallCtx<'a> { + args: &'a [u64], + ctx: *mut vm::Ctx, + callable: NonNull, + } + extern "C" fn call_fn(f: *mut u8) -> u64 { + unsafe { + let f = &*(f as *const CallCtx); + let callable: extern "C" fn( + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + ) + -> u64 = std::mem::transmute(f.callable); + let mut args = f.args.iter(); + callable( + f.ctx as u64, + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + ) + } + } + let mut cctx = CallCtx { + args: &args, + ctx: ctx, + callable: func, + }; + use libc::{ + mmap, munmap, MAP_ANON, MAP_NORESERVE, MAP_PRIVATE, PROT_READ, + PROT_WRITE, + }; + const STACK_SIZE: usize = 1048576 * 1024; // 1GB of virtual address space for stack. + let stack_ptr = mmap( + ::std::ptr::null_mut(), + STACK_SIZE, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, + -1, + 0, + ); + if stack_ptr as isize == -1 { + panic!("unable to allocate stack"); + } + // TODO: Mark specific regions in the stack as PROT_NONE. + let ret = SWITCH_STACK( + (stack_ptr as *mut u8).offset(STACK_SIZE as isize) as *mut u64, + call_fn, + &mut cctx as *mut CallCtx as *mut u8, + ); + munmap(stack_ptr, STACK_SIZE); + ret + } + }, + rm.get_breakpoints(), + ) { + Ok(x) => { + if !rets.is_null() { + *rets = x; + } + true + } + Err(err) => { + *error_out = Some(err); + false + } + }; + ret + } + + unsafe extern "C" fn dummy_trampoline( + _: *mut vm::Ctx, + _: NonNull, + _: *const u64, + _: *mut u64, + ) { + unreachable!() + } + + Some(unsafe { + Wasm::from_raw_parts( + dummy_trampoline, + invoke, + NonNull::new((self.signatures.get(sig_index).unwrap().params().len() + 1) as _), // +1 to keep it non-zero + ) + }) + } + + unsafe fn do_early_trap(&self, data: Box) -> ! { + fault::begin_unsafe_unwind(data); + } + + fn get_code(&self) -> Option<&[u8]> { + Some(&self.code) + } + + fn get_offsets(&self) -> Option> { + Some(self.function_offsets.iter().map(|x| x.0).collect()) + } + + fn get_local_function_offsets(&self) -> Option> { + Some( + self.function_offsets[self.func_import_count..] + .iter() + .map(|x| x.0) + .collect(), + ) + } + + /// Returns the inline breakpoint size corresponding to an Architecture. + fn get_inline_breakpoint_size(&self, arch: Architecture) -> Option { + match arch { + Architecture::X64 => Some(INLINE_BREAKPOINT_SIZE_X86_SINGLEPASS), + Architecture::Aarch64 => Some(INLINE_BREAKPOINT_SIZE_AARCH64_SINGLEPASS), + } + } + + /// Attempts to read an inline breakpoint from the code. + /// + /// Inline breakpoints are detected by special instruction sequences that never + /// appear in valid code. + fn read_inline_breakpoint(&self, arch: Architecture, code: &[u8]) -> Option { + match arch { + Architecture::X64 => { + if code.len() < INLINE_BREAKPOINT_SIZE_X86_SINGLEPASS { + None + } else if &code[..INLINE_BREAKPOINT_SIZE_X86_SINGLEPASS - 1] + == &[ + 0x0f, 0x0b, // ud2 + 0x0f, 0xb9, // ud + 0xcd, 0xff, // int 0xff + ] + { + Some(InlineBreakpoint { + size: INLINE_BREAKPOINT_SIZE_X86_SINGLEPASS, + ty: match code[INLINE_BREAKPOINT_SIZE_X86_SINGLEPASS - 1] { + 0 => InlineBreakpointType::Middleware, + _ => return None, + }, + }) + } else { + None + } + } + Architecture::Aarch64 => { + if code.len() < INLINE_BREAKPOINT_SIZE_AARCH64_SINGLEPASS { + None + } else if &code[..INLINE_BREAKPOINT_SIZE_AARCH64_SINGLEPASS - 4] + == &[ + 0, 0, 0, 0, // udf #0 + 0xff, 0xff, 0x00, 0x00, // udf #65535 + ] + { + Some(InlineBreakpoint { + size: INLINE_BREAKPOINT_SIZE_AARCH64_SINGLEPASS, + ty: match code[INLINE_BREAKPOINT_SIZE_AARCH64_SINGLEPASS - 4] { + 0 => InlineBreakpointType::Middleware, + _ => return None, + }, + }) + } else { + None + } + } + } + } +} + +#[derive(Debug)] +pub struct CodegenError { + pub message: String, +} + +#[derive(Copy, Clone, Debug)] +struct CodegenConfig { + memory_bound_check_mode: MemoryBoundCheckMode, + enforce_stack_check: bool, + track_state: bool, + full_preemption: bool, +} + +impl ModuleCodeGenerator + for X64ModuleCodeGenerator +{ + fn new() -> X64ModuleCodeGenerator { + let a = Assembler::new().unwrap(); + + X64ModuleCodeGenerator { + functions: vec![], + signatures: None, + function_signatures: None, + function_labels: Some(HashMap::new()), + assembler: Some(a), + func_import_count: 0, + config: None, + } + } + + /// Singlepass does validation as it compiles + fn requires_pre_validation() -> bool { + true + } + + fn backend_id() -> &'static str { + BACKEND_ID + } + + fn new_with_target(_: Option, _: Option, _: Option) -> Self { + unimplemented!("cross compilation is not available for singlepass backend") + } + + fn check_precondition(&mut self, _module_info: &ModuleInfo) -> Result<(), CodegenError> { + Ok(()) + } + + fn next_function( + &mut self, + _module_info: Arc>, + _loc: WasmSpan, + ) -> Result<&mut X64FunctionCode, CodegenError> { + let (mut assembler, mut function_labels, breakpoints, exception_table) = + match self.functions.last_mut() { + Some(x) => ( + x.assembler.take().unwrap(), + x.function_labels.take().unwrap(), + x.breakpoints.take().unwrap(), + x.exception_table.take(), + ), + None => ( + self.assembler.take().unwrap(), + self.function_labels.take().unwrap(), + HashMap::new(), + None, // Some(ExceptionTable::new()), + ), + }; + + let begin_offset = assembler.offset(); + let begin_label_info = function_labels + .entry(self.functions.len() + self.func_import_count) + .or_insert_with(|| (assembler.new_dynamic_label(), None)); + + begin_label_info.1 = Some(begin_offset); + assembler.arch_emit_entry_trampoline(); + let begin_label = begin_label_info.0; + let mut machine = Machine::new(); + machine.track_state = self.config.as_ref().unwrap().track_state; + + assembler.emit_label(begin_label); + let code = X64FunctionCode { + local_function_id: self.functions.len(), + + signatures: self.signatures.as_ref().unwrap().clone(), + function_signatures: self.function_signatures.as_ref().unwrap().clone(), + fsm: FunctionStateMap::new(new_machine_state(), self.functions.len(), 32, vec![]), // only a placeholder; this is initialized later in `begin_body` + offset: begin_offset.0, + + assembler: Some(assembler), + function_labels: Some(function_labels), + breakpoints: Some(breakpoints), + returns: smallvec![], + locals: vec![], + num_params: 0, + num_locals: 0, + value_stack: vec![], + control_stack: vec![], + machine, + unreachable_depth: 0, + config: self.config.as_ref().unwrap().clone(), + exception_table: exception_table, + }; + self.functions.push(code); + Ok(self.functions.last_mut().unwrap()) + } + + fn finalize( + mut self, + _: &ModuleInfo, + ) -> Result< + ( + X64ExecutionContext, + Option, + Box, + ), + CodegenError, + > { + let (assembler, function_labels, breakpoints, exception_table) = + match self.functions.last_mut() { + Some(x) => ( + x.assembler.take().unwrap(), + x.function_labels.take().unwrap(), + x.breakpoints.take().unwrap(), + x.exception_table.take(), + ), + None => ( + self.assembler.take().unwrap(), + self.function_labels.take().unwrap(), + HashMap::new(), + None, // Some(ExceptionTable::new()), + ), + }; + + let total_size = assembler.get_offset().0; + let _output = assembler.finalize().unwrap(); + let mut output = CodeMemory::new(_output.len()); + output[0.._output.len()].copy_from_slice(&_output); + output.make_executable(); + + let mut out_labels: Vec = vec![]; + let mut out_offsets: Vec = vec![]; + + for i in 0..function_labels.len() { + let (_, offset) = match function_labels.get(&i) { + Some(x) => x, + None => { + return Err(CodegenError { + message: format!("label not found"), + }); + } + }; + let offset = match offset { + Some(x) => x, + None => { + return Err(CodegenError { + message: format!("offset is none"), + }); + } + }; + out_labels.push(FuncPtr( + unsafe { output.as_ptr().offset(offset.0 as isize) } as _, + )); + out_offsets.push(*offset); + } + + let breakpoints: Arc> = Arc::new( + breakpoints + .into_iter() + .map(|(offset, f)| { + ( + unsafe { output.as_ptr().offset(offset.0 as isize) } as usize, + f, + ) + }) + .collect(), + ); + + let local_function_maps: BTreeMap = self + .functions + .iter() + .map(|x| (x.offset, x.fsm.clone())) + .collect(); + + let msm = ModuleStateMap { + local_functions: local_function_maps, + total_size, + }; + + let cache_image = CacheImage { + code: output.to_vec(), + function_pointers: out_labels + .iter() + .map(|x| { + (x.0 as usize) + .checked_sub(output.as_ptr() as usize) + .unwrap() + }) + .collect(), + function_offsets: out_offsets.iter().map(|x| x.0 as usize).collect(), + func_import_count: self.func_import_count, + msm: msm.clone(), + exception_table: exception_table.clone(), + }; + + let cache = if unsafe { USE_RKYV_SERIALIZATION } { + let mut serializer = AllocSerializer::<4096>::default(); + serializer.serialize_value(&cache_image).unwrap(); + let archived_cache_image = serializer.into_serializer().into_inner(); + + SinglepassCache { + buffer: Arc::from(archived_cache_image.as_slice()), + } + } else { + let serialized_cache_image = bincode::serialize(&cache_image).unwrap().into_boxed_slice(); + SinglepassCache { + buffer: Arc::from(serialized_cache_image), + } + }; + + Ok(( + X64ExecutionContext { + code: output, + signatures: self.signatures.as_ref().unwrap().clone(), + breakpoints: breakpoints, + func_import_count: self.func_import_count, + function_pointers: out_labels, + function_offsets: out_offsets, + msm: msm, + exception_table: exception_table, + }, + None, + Box::new(cache), + )) + } + + fn feed_signatures(&mut self, signatures: Map) -> Result<(), CodegenError> { + self.signatures = Some(Arc::new(signatures)); + Ok(()) + } + + fn feed_function_signatures( + &mut self, + assoc: Map, + ) -> Result<(), CodegenError> { + self.function_signatures = Some(Arc::new(assoc)); + Ok(()) + } + + fn feed_import_function(&mut self) -> Result<(), CodegenError> { + let labels = self.function_labels.as_mut().unwrap(); + let id = labels.len(); + + let a = self.assembler.as_mut().unwrap(); + let offset = a.offset(); + a.arch_emit_entry_trampoline(); + let label = a.get_label(); + a.emit_label(label); + labels.insert(id, (label, Some(offset))); + + // Emits a tail call trampoline that loads the address of the target import function + // from Ctx and jumps to it. + + let imported_funcs_addr = vm::Ctx::offset_imported_funcs(); + let imported_func = vm::ImportedFunc::size() as usize * id; + let imported_func_addr = imported_func + vm::ImportedFunc::offset_func() as usize; + let imported_func_ctx_addr = imported_func + vm::ImportedFunc::offset_func_ctx() as usize; + let imported_func_ctx_vmctx_addr = vm::FuncCtx::offset_vmctx() as usize; + + a.emit_mov( + Size::S64, + Location::Memory(GPR::RDI, imported_funcs_addr as i32), + Location::GPR(GPR::RAX), + ); + a.emit_mov( + Size::S64, + Location::Memory(GPR::RAX, imported_func_ctx_addr as i32), + Location::GPR(GPR::RDI), + ); + a.emit_mov( + Size::S64, + Location::Memory(GPR::RDI, imported_func_ctx_vmctx_addr as i32), + Location::GPR(GPR::RDI), + ); + a.emit_mov( + Size::S64, + Location::Memory(GPR::RAX, imported_func_addr as i32), + Location::GPR(GPR::RAX), + ); + a.emit_host_redirection(GPR::RAX); + + self.func_import_count += 1; + + Ok(()) + } + + fn feed_compiler_config(&mut self, config: &CompilerConfig) -> Result<(), CodegenError> { + self.config = Some(Arc::new(CodegenConfig { + memory_bound_check_mode: config.memory_bound_check_mode, + enforce_stack_check: config.enforce_stack_check, + track_state: config.track_state, + full_preemption: config.full_preemption, + })); + Ok(()) + } + unsafe fn from_cache(artifact: Artifact, _: Token) -> Result { + let (info, _, memory) = artifact.consume(); + + let cache_image: CacheImage = if USE_RKYV_SERIALIZATION { + let memory_contents = memory.as_slice_contents(); + let archived_cache_image: &Archived + = rkyv::archived_root::(memory_contents); + RkyvDeserialize::::deserialize(archived_cache_image, &mut rkyv::Infallible).unwrap() + } else { + bincode::deserialize(memory.as_slice()) + .map_err(|x| CacheError::DeserializeError(format!("{:?}", x)))? + }; + + let mut code_mem = CodeMemory::new(cache_image.code.len()); + code_mem[0..cache_image.code.len()].copy_from_slice(&cache_image.code); + code_mem.make_executable(); + + let function_pointers: Vec = cache_image + .function_pointers + .iter() + .map(|&x| FuncPtr(code_mem.as_ptr().offset(x as isize) as *const FuncPtrInner)) + .collect(); + let function_offsets: Vec = cache_image + .function_offsets + .iter() + .cloned() + .map(AssemblyOffset) + .collect(); + + let ec = X64ExecutionContext { + code: code_mem, + function_pointers, + function_offsets, + signatures: Arc::new(info.signatures.clone()), + breakpoints: Arc::new(HashMap::new()), + func_import_count: cache_image.func_import_count, + msm: cache_image.msm, + exception_table: cache_image.exception_table, + }; + Ok(ModuleInner { + runnable_module: Arc::new(Box::new(ec)), + cache_gen: Box::new(SinglepassCache { + buffer: Arc::from(memory.as_slice().to_vec().into_boxed_slice()), + }), + info, + }) + } +} + +impl X64FunctionCode { + fn mark_trappable( + a: &mut Assembler, + m: &Machine, + fsm: &mut FunctionStateMap, + control_stack: &mut [ControlFrame], + ) { + let state_diff_id = Self::get_state_diff(m, fsm, control_stack); + let offset = a.get_offset().0; + fsm.trappable_offsets.insert( + offset, + OffsetInfo { + end_offset: offset + 1, + activate_offset: offset, + diff_id: state_diff_id, + }, + ); + fsm.wasm_offset_to_target_offset + .insert(m.state.wasm_inst_offset, SuspendOffset::Trappable(offset)); + } + + /// Marks each address in the code range emitted by `f` with the exception code `code`. + fn mark_range_with_exception_code R, R>( + a: &mut Assembler, + etable: &mut ExceptionTable, + code: ExceptionCode, + f: F, + ) -> R { + let begin = a.get_offset().0; + let ret = f(a); + let end = a.get_offset().0; + for i in begin..end { + etable.offset_to_code.insert(i, code); + } + ret + } + + /// Moves `loc` to a valid location for `div`/`idiv`. + fn emit_relaxed_xdiv( + a: &mut Assembler, + m: &mut Machine, + exception_table: &mut Option, + op: fn(&mut Assembler, Size, Location), + sz: Size, + loc: Location, + fsm: &mut FunctionStateMap, + control_stack: &mut [ControlFrame], + ) { + m.state.wasm_stack_private_depth += 1; + match loc { + Location::Imm64(_) | Location::Imm32(_) => { + a.emit_mov(sz, loc, Location::GPR(GPR::RCX)); // must not be used during div (rax, rdx) + Self::mark_trappable(a, m, fsm, control_stack); + + match exception_table { + Some(etable) => { etable + .offset_to_code + .insert(a.get_offset().0, ExceptionCode::IllegalArithmetic); } + None => {} + }; + + op(a, sz, Location::GPR(GPR::RCX)); + } + _ => { + Self::mark_trappable(a, m, fsm, control_stack); + + match exception_table { + Some(etable) => { etable + .offset_to_code + .insert(a.get_offset().0, ExceptionCode::IllegalArithmetic); } + None => {} + }; + + op(a, sz, loc); + } + } + m.state.wasm_stack_private_depth -= 1; + } + + /// Moves `src` and `dst` to valid locations for `movzx`/`movsx`. + fn emit_relaxed_zx_sx( + a: &mut Assembler, + m: &mut Machine, + op: fn(&mut Assembler, Size, Location, Size, Location), + sz_src: Size, + mut src: Location, + sz_dst: Size, + dst: Location, + ) -> Result<(), CodegenError> { + let inner = |m: &mut Machine, a: &mut Assembler, src: Location| match dst { + Location::Imm32(_) | Location::Imm64(_) => { + return Err(CodegenError { + message: format!("emit_relaxed_zx_sx dst Imm: unreachable code"), + }) + } + Location::Memory(_, _) => { + let tmp_dst = m.acquire_temp_gpr().unwrap(); + op(a, sz_src, src, sz_dst, Location::GPR(tmp_dst)); + a.emit_mov(Size::S64, Location::GPR(tmp_dst), dst); + + m.release_temp_gpr(tmp_dst); + Ok(()) + } + Location::GPR(_) => { + op(a, sz_src, src, sz_dst, dst); + Ok(()) + } + _ => { + return Err(CodegenError { + message: format!("emit_relaxed_zx_sx dst: unreachable code"), + }) + } + }; + + match src { + Location::Imm32(_) | Location::Imm64(_) => { + let tmp_src = m.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S64, src, Location::GPR(tmp_src)); + src = Location::GPR(tmp_src); + + inner(m, a, src)?; + + m.release_temp_gpr(tmp_src); + } + Location::GPR(_) | Location::Memory(_, _) => inner(m, a, src)?, + _ => { + return Err(CodegenError { + message: format!("emit_relaxed_zx_sx src: unreachable code"), + }) + } + } + Ok(()) + } + + /// Moves `src` and `dst` to valid locations for generic instructions. + fn emit_relaxed_binop( + a: &mut Assembler, + m: &mut Machine, + op: fn(&mut Assembler, Size, Location, Location), + sz: Size, + src: Location, + dst: Location, + ) { + enum RelaxMode { + Direct, + SrcToGPR, + DstToGPR, + BothToGPR, + } + let mode = match (src, dst) { + (Location::GPR(_), Location::GPR(_)) + if (op as *const u8 == Assembler::emit_imul as *const u8) => + { + RelaxMode::Direct + } + _ if (op as *const u8 == Assembler::emit_imul as *const u8) => RelaxMode::BothToGPR, + + (Location::Memory(_, _), Location::Memory(_, _)) => RelaxMode::SrcToGPR, + (Location::Imm64(_), Location::Imm64(_)) | (Location::Imm64(_), Location::Imm32(_)) => { + RelaxMode::BothToGPR + } + (_, Location::Imm32(_)) | (_, Location::Imm64(_)) => RelaxMode::DstToGPR, + (Location::Imm64(_), Location::Memory(_, _)) => RelaxMode::SrcToGPR, + (Location::Imm64(_), Location::GPR(_)) + if (op as *const u8 != Assembler::emit_mov as *const u8) => + { + RelaxMode::SrcToGPR + } + (_, Location::XMM(_)) => RelaxMode::SrcToGPR, + _ => RelaxMode::Direct, + }; + + match mode { + RelaxMode::SrcToGPR => { + let temp = m.acquire_temp_gpr().unwrap(); + a.emit_mov(sz, src, Location::GPR(temp)); + op(a, sz, Location::GPR(temp), dst); + m.release_temp_gpr(temp); + } + RelaxMode::DstToGPR => { + let temp = m.acquire_temp_gpr().unwrap(); + a.emit_mov(sz, dst, Location::GPR(temp)); + op(a, sz, src, Location::GPR(temp)); + m.release_temp_gpr(temp); + } + RelaxMode::BothToGPR => { + let temp_src = m.acquire_temp_gpr().unwrap(); + let temp_dst = m.acquire_temp_gpr().unwrap(); + a.emit_mov(sz, src, Location::GPR(temp_src)); + a.emit_mov(sz, dst, Location::GPR(temp_dst)); + op(a, sz, Location::GPR(temp_src), Location::GPR(temp_dst)); + match dst { + Location::Memory(_, _) | Location::GPR(_) => { + a.emit_mov(sz, Location::GPR(temp_dst), dst); + } + _ => {} + } + m.release_temp_gpr(temp_dst); + m.release_temp_gpr(temp_src); + } + RelaxMode::Direct => { + op(a, sz, src, dst); + } + } + } + + /// Moves `src1` and `src2` to valid locations and possibly adds a layer of indirection for `dst` for AVX instructions. + fn emit_relaxed_avx( + a: &mut Assembler, + m: &mut Machine, + op: fn(&mut Assembler, XMM, XMMOrMemory, XMM), + src1: Location, + src2: Location, + dst: Location, + ) -> Result<(), CodegenError> { + Self::emit_relaxed_avx_base( + a, + m, + |a, _, src1, src2, dst| op(a, src1, src2, dst), + src1, + src2, + dst, + )?; + Ok(()) + } + + /// Moves `src1` and `src2` to valid locations and possibly adds a layer of indirection for `dst` for AVX instructions. + fn emit_relaxed_avx_base( + a: &mut Assembler, + m: &mut Machine, + op: F, + src1: Location, + src2: Location, + dst: Location, + ) -> Result<(), CodegenError> { + let tmp1 = m.acquire_temp_xmm().unwrap(); + let tmp2 = m.acquire_temp_xmm().unwrap(); + let tmp3 = m.acquire_temp_xmm().unwrap(); + let tmpg = m.acquire_temp_gpr().unwrap(); + + let src1 = match src1 { + Location::XMM(x) => x, + Location::GPR(_) | Location::Memory(_, _) => { + a.emit_mov(Size::S64, src1, Location::XMM(tmp1)); + tmp1 + } + Location::Imm32(_) => { + a.emit_mov(Size::S32, src1, Location::GPR(tmpg)); + a.emit_mov(Size::S32, Location::GPR(tmpg), Location::XMM(tmp1)); + tmp1 + } + Location::Imm64(_) => { + a.emit_mov(Size::S64, src1, Location::GPR(tmpg)); + a.emit_mov(Size::S64, Location::GPR(tmpg), Location::XMM(tmp1)); + tmp1 + } + _ => { + return Err(CodegenError { + message: format!("emit_relaxed_avx_base src1: unreachable code"), + }) + } + }; + + let src2 = match src2 { + Location::XMM(x) => XMMOrMemory::XMM(x), + Location::Memory(base, disp) => XMMOrMemory::Memory(base, disp), + Location::GPR(_) => { + a.emit_mov(Size::S64, src2, Location::XMM(tmp2)); + XMMOrMemory::XMM(tmp2) + } + Location::Imm32(_) => { + a.emit_mov(Size::S32, src2, Location::GPR(tmpg)); + a.emit_mov(Size::S32, Location::GPR(tmpg), Location::XMM(tmp2)); + XMMOrMemory::XMM(tmp2) + } + Location::Imm64(_) => { + a.emit_mov(Size::S64, src2, Location::GPR(tmpg)); + a.emit_mov(Size::S64, Location::GPR(tmpg), Location::XMM(tmp2)); + XMMOrMemory::XMM(tmp2) + } + _ => { + return Err(CodegenError { + message: format!("emit_relaxed_avx_base src2: unreachable code"), + }) + } + }; + + match dst { + Location::XMM(x) => { + op(a, m, src1, src2, x); + } + Location::Memory(_, _) | Location::GPR(_) => { + op(a, m, src1, src2, tmp3); + a.emit_mov(Size::S64, Location::XMM(tmp3), dst); + } + _ => { + return Err(CodegenError { + message: format!("emit_relaxed_avx_base dst: unreachable code"), + }) + } + } + + m.release_temp_gpr(tmpg); + m.release_temp_xmm(tmp3); + m.release_temp_xmm(tmp2); + m.release_temp_xmm(tmp1); + Ok(()) + } + + /// I32 binary operation with both operands popped from the virtual stack. + fn emit_binop_i32( + a: &mut Assembler, + m: &mut Machine, + value_stack: &mut Vec, + f: fn(&mut Assembler, Size, Location, Location), + ) { + // Using Red Zone here. + let loc_b = get_location_released(a, m, value_stack.pop().unwrap()); + let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); + let ret = m.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; + + if loc_a != ret { + let tmp = m.acquire_temp_gpr().unwrap(); + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S32, + loc_a, + Location::GPR(tmp), + ); + Self::emit_relaxed_binop(a, m, f, Size::S32, loc_b, Location::GPR(tmp)); + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S32, + Location::GPR(tmp), + ret, + ); + m.release_temp_gpr(tmp); + } else { + Self::emit_relaxed_binop(a, m, f, Size::S32, loc_b, ret); + } + + value_stack.push(ret); + } + + /// I64 binary operation with both operands popped from the virtual stack. + fn emit_binop_i64( + a: &mut Assembler, + m: &mut Machine, + value_stack: &mut Vec, + f: fn(&mut Assembler, Size, Location, Location), + ) { + // Using Red Zone here. + let loc_b = get_location_released(a, m, value_stack.pop().unwrap()); + let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); + let ret = m.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; + + if loc_a != ret { + let tmp = m.acquire_temp_gpr().unwrap(); + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S64, + loc_a, + Location::GPR(tmp), + ); + Self::emit_relaxed_binop(a, m, f, Size::S64, loc_b, Location::GPR(tmp)); + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S64, + Location::GPR(tmp), + ret, + ); + m.release_temp_gpr(tmp); + } else { + Self::emit_relaxed_binop(a, m, f, Size::S64, loc_b, ret); + } + + value_stack.push(ret); + } + + /// I32 comparison with `loc_b` from input. + fn emit_cmpop_i32_dynamic_b( + a: &mut Assembler, + m: &mut Machine, + value_stack: &mut Vec, + c: Condition, + loc_b: Location, + ) -> Result<(), CodegenError> { + // Using Red Zone here. + let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); + + let ret = m.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; + match ret { + Location::GPR(x) => { + Self::emit_relaxed_binop(a, m, Assembler::emit_cmp, Size::S32, loc_b, loc_a); + a.emit_set(c, x); + a.emit_and(Size::S32, Location::Imm32(0xff), Location::GPR(x)); + } + Location::Memory(_, _) => { + let tmp = m.acquire_temp_gpr().unwrap(); + Self::emit_relaxed_binop(a, m, Assembler::emit_cmp, Size::S32, loc_b, loc_a); + a.emit_set(c, tmp); + a.emit_and(Size::S32, Location::Imm32(0xff), Location::GPR(tmp)); + a.emit_mov(Size::S32, Location::GPR(tmp), ret); + m.release_temp_gpr(tmp); + } + _ => { + return Err(CodegenError { + message: format!("emit_cmpop_i32_dynamic_b ret: unreachable code"), + }) + } + } + value_stack.push(ret); + Ok(()) + } + + /// I32 comparison with both operands popped from the virtual stack. + fn emit_cmpop_i32( + a: &mut Assembler, + m: &mut Machine, + value_stack: &mut Vec, + c: Condition, + ) -> Result<(), CodegenError> { + let loc_b = get_location_released(a, m, value_stack.pop().unwrap()); + Self::emit_cmpop_i32_dynamic_b(a, m, value_stack, c, loc_b)?; + Ok(()) + } + + /// I64 comparison with `loc_b` from input. + fn emit_cmpop_i64_dynamic_b( + a: &mut Assembler, + m: &mut Machine, + value_stack: &mut Vec, + c: Condition, + loc_b: Location, + ) -> Result<(), CodegenError> { + // Using Red Zone here. + let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); + + let ret = m.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; + match ret { + Location::GPR(x) => { + Self::emit_relaxed_binop(a, m, Assembler::emit_cmp, Size::S64, loc_b, loc_a); + a.emit_set(c, x); + a.emit_and(Size::S32, Location::Imm32(0xff), Location::GPR(x)); + } + Location::Memory(_, _) => { + let tmp = m.acquire_temp_gpr().unwrap(); + Self::emit_relaxed_binop(a, m, Assembler::emit_cmp, Size::S64, loc_b, loc_a); + a.emit_set(c, tmp); + a.emit_and(Size::S32, Location::Imm32(0xff), Location::GPR(tmp)); + a.emit_mov(Size::S32, Location::GPR(tmp), ret); + m.release_temp_gpr(tmp); + } + _ => { + return Err(CodegenError { + message: format!("emit_cmpop_i64_dynamic_b ret: unreachable code"), + }) + } + } + value_stack.push(ret); + Ok(()) + } + + /// I64 comparison with both operands popped from the virtual stack. + fn emit_cmpop_i64( + a: &mut Assembler, + m: &mut Machine, + value_stack: &mut Vec, + c: Condition, + ) -> Result<(), CodegenError> { + let loc_b = get_location_released(a, m, value_stack.pop().unwrap()); + Self::emit_cmpop_i64_dynamic_b(a, m, value_stack, c, loc_b)?; + Ok(()) + } + + /// I32 `lzcnt`/`tzcnt`/`popcnt` with operand popped from the virtual stack. + fn emit_xcnt_i32( + a: &mut Assembler, + m: &mut Machine, + value_stack: &mut Vec, + f: fn(&mut Assembler, Size, Location, Location), + ) -> Result<(), CodegenError> { + let loc = get_location_released(a, m, value_stack.pop().unwrap()); + let ret = m.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; + + match loc { + Location::Imm32(_) => { + let tmp = m.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S32, loc, Location::GPR(tmp)); + if let Location::Memory(_, _) = ret { + let out_tmp = m.acquire_temp_gpr().unwrap(); + f(a, Size::S32, Location::GPR(tmp), Location::GPR(out_tmp)); + a.emit_mov(Size::S32, Location::GPR(out_tmp), ret); + m.release_temp_gpr(out_tmp); + } else { + f(a, Size::S32, Location::GPR(tmp), ret); + } + m.release_temp_gpr(tmp); + } + Location::Memory(_, _) | Location::GPR(_) => { + if let Location::Memory(_, _) = ret { + let out_tmp = m.acquire_temp_gpr().unwrap(); + f(a, Size::S32, loc, Location::GPR(out_tmp)); + a.emit_mov(Size::S32, Location::GPR(out_tmp), ret); + m.release_temp_gpr(out_tmp); + } else { + f(a, Size::S32, loc, ret); + } + } + _ => { + return Err(CodegenError { + message: format!("emit_xcnt_i32 loc: unreachable code"), + }) + } + } + value_stack.push(ret); + Ok(()) + } + + /// I64 `lzcnt`/`tzcnt`/`popcnt` with operand popped from the virtual stack. + fn emit_xcnt_i64( + a: &mut Assembler, + m: &mut Machine, + value_stack: &mut Vec, + f: fn(&mut Assembler, Size, Location, Location), + ) -> Result<(), CodegenError> { + let loc = get_location_released(a, m, value_stack.pop().unwrap()); + let ret = m.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; + + match loc { + Location::Imm64(_) | Location::Imm32(_) => { + let tmp = m.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S64, loc, Location::GPR(tmp)); + if let Location::Memory(_, _) = ret { + let out_tmp = m.acquire_temp_gpr().unwrap(); + f(a, Size::S64, Location::GPR(tmp), Location::GPR(out_tmp)); + a.emit_mov(Size::S64, Location::GPR(out_tmp), ret); + m.release_temp_gpr(out_tmp); + } else { + f(a, Size::S64, Location::GPR(tmp), ret); + } + m.release_temp_gpr(tmp); + } + Location::Memory(_, _) | Location::GPR(_) => { + if let Location::Memory(_, _) = ret { + let out_tmp = m.acquire_temp_gpr().unwrap(); + f(a, Size::S64, loc, Location::GPR(out_tmp)); + a.emit_mov(Size::S64, Location::GPR(out_tmp), ret); + m.release_temp_gpr(out_tmp); + } else { + f(a, Size::S64, loc, ret); + } + } + _ => { + return Err(CodegenError { + message: format!("emit_xcnt_i64 loc: unreachable code"), + }) + } + } + value_stack.push(ret); + Ok(()) + } + + /// I32 shift with both operands popped from the virtual stack. + fn emit_shift_i32( + a: &mut Assembler, + m: &mut Machine, + value_stack: &mut Vec, + f: fn(&mut Assembler, Size, Location, Location), + ) { + let loc_b = get_location_released(a, m, value_stack.pop().unwrap()); + let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); + let ret = m.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; + + a.emit_mov(Size::S32, loc_b, Location::GPR(GPR::RCX)); + + if loc_a != ret { + Self::emit_relaxed_binop(a, m, Assembler::emit_mov, Size::S32, loc_a, ret); + } + + f(a, Size::S32, Location::GPR(GPR::RCX), ret); + value_stack.push(ret); + } + + /// I64 shift with both operands popped from the virtual stack. + fn emit_shift_i64( + a: &mut Assembler, + m: &mut Machine, + value_stack: &mut Vec, + f: fn(&mut Assembler, Size, Location, Location), + ) { + let loc_b = get_location_released(a, m, value_stack.pop().unwrap()); + let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); + let ret = m.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; + + a.emit_mov(Size::S64, loc_b, Location::GPR(GPR::RCX)); + + if loc_a != ret { + Self::emit_relaxed_binop(a, m, Assembler::emit_mov, Size::S64, loc_a, ret); + } + + f(a, Size::S64, Location::GPR(GPR::RCX), ret); + value_stack.push(ret); + } + + /// Floating point (AVX) binary operation with both operands popped from the virtual stack. + fn emit_fp_binop_avx( + a: &mut Assembler, + m: &mut Machine, + value_stack: &mut Vec, + f: fn(&mut Assembler, XMM, XMMOrMemory, XMM), + ) -> Result<(), CodegenError> { + let loc_b = get_location_released(a, m, value_stack.pop().unwrap()); + let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); + let ret = m.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; + value_stack.push(ret); + + Self::emit_relaxed_avx(a, m, f, loc_a, loc_b, ret)?; + Ok(()) + } + + /// Floating point (AVX) comparison with both operands popped from the virtual stack. + fn emit_fp_cmpop_avx( + a: &mut Assembler, + m: &mut Machine, + value_stack: &mut Vec, + f: fn(&mut Assembler, XMM, XMMOrMemory, XMM), + ) -> Result<(), CodegenError> { + let loc_b = get_location_released(a, m, value_stack.pop().unwrap()); + let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); + let ret = m.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; + value_stack.push(ret); + + Self::emit_relaxed_avx(a, m, f, loc_a, loc_b, ret)?; + + // Workaround for behavior inconsistency among different backing implementations. + // (all bits or only the least significant bit are set to one?) + a.emit_and(Size::S32, Location::Imm32(1), ret); + Ok(()) + } + + /// Floating point (AVX) binary operation with both operands popped from the virtual stack. + fn emit_fp_unop_avx( + a: &mut Assembler, + m: &mut Machine, + value_stack: &mut Vec, + f: fn(&mut Assembler, XMM, XMMOrMemory, XMM), + ) -> Result<(), CodegenError> { + let loc = get_location_released(a, m, value_stack.pop().unwrap()); + let ret = m.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; + value_stack.push(ret); + + Self::emit_relaxed_avx(a, m, f, loc, loc, ret)?; + Ok(()) + } + + /// Emits a System V call sequence. + /// + /// This function must not use RAX before `cb` is called. + fn emit_call_sysv, F: FnOnce(&mut Assembler)>( + a: &mut Assembler, + m: &mut Machine, + cb: F, + params: I, + state_context: Option<(&mut FunctionStateMap, &mut [ControlFrame])>, + ) -> Result<(), CodegenError> { + // Values pushed in this function are above the shadow region. + m.state.stack_values.push(MachineValue::ExplicitShadow); + + let params: Vec<_> = params.collect(); + + // Save used GPRs. + let used_gprs = m.get_used_gprs(); + for r in used_gprs.iter() { + a.emit_push(Size::S64, Location::GPR(*r)); + let content = m.state.register_values[X64Register::GPR(*r).to_index().0].clone(); + if content == MachineValue::Undefined { + return Err(CodegenError { + message: format!("emit_call_sysv: Undefined used_gprs content"), + }); + } + m.state.stack_values.push(content); + } + + // Save used XMM registers. + let used_xmms = m.get_used_xmms(); + if used_xmms.len() > 0 { + a.emit_sub( + Size::S64, + Location::Imm32((used_xmms.len() * 8) as u32), + Location::GPR(GPR::RSP), + ); + + for (i, r) in used_xmms.iter().enumerate() { + a.emit_mov( + Size::S64, + Location::XMM(*r), + Location::Memory(GPR::RSP, (i * 8) as i32), + ); + } + for r in used_xmms.iter().rev() { + let content = m.state.register_values[X64Register::XMM(*r).to_index().0].clone(); + if content == MachineValue::Undefined { + return Err(CodegenError { + message: format!("emit_call_sysv: Undefined used_xmms content"), + }); + } + m.state.stack_values.push(content); + } + } + + let mut stack_offset: usize = 0; + + // Calculate stack offset. + for (i, _param) in params.iter().enumerate() { + let loc = Machine::get_param_location(1 + i); + match loc { + Location::Memory(_, _) => { + stack_offset += 8; + } + _ => {} + } + } + + // Align stack to 16 bytes. + if (m.get_stack_offset() + used_gprs.len() * 8 + used_xmms.len() * 8 + stack_offset) % 16 + != 0 + { + a.emit_sub(Size::S64, Location::Imm32(8), Location::GPR(GPR::RSP)); + stack_offset += 8; + m.state.stack_values.push(MachineValue::Undefined); + } + + let mut call_movs: Vec<(Location, GPR)> = vec![]; + + // Prepare register & stack parameters. + for (i, param) in params.iter().enumerate().rev() { + let loc = Machine::get_param_location(1 + i); + match loc { + Location::GPR(x) => { + call_movs.push((*param, x)); + } + Location::Memory(_, _) => { + match *param { + Location::GPR(x) => { + let content = + m.state.register_values[X64Register::GPR(x).to_index().0].clone(); + // FIXME: There might be some corner cases (release -> emit_call_sysv -> acquire?) that cause this assertion to fail. + // Hopefully nothing would be incorrect at runtime. + + //assert!(content != MachineValue::Undefined); + m.state.stack_values.push(content); + } + Location::XMM(x) => { + let content = + m.state.register_values[X64Register::XMM(x).to_index().0].clone(); + //assert!(content != MachineValue::Undefined); + m.state.stack_values.push(content); + } + Location::Memory(reg, offset) => { + if reg != GPR::RBP { + return Err(CodegenError { + message: format!("emit_call_sysv loc param: unreachable code"), + }); + } + m.state + .stack_values + .push(MachineValue::CopyStackBPRelative(offset)); + // TODO: Read value at this offset + } + _ => { + m.state.stack_values.push(MachineValue::Undefined); + } + } + match *param { + Location::Imm64(_) => { + // Dummy value slot to be filled with `mov`. + a.emit_push(Size::S64, Location::GPR(GPR::RAX)); + + // Use R10 as the temporary register here, since it is callee-saved and not + // used by the callback `cb`. + a.emit_mov(Size::S64, *param, Location::GPR(GPR::R10)); + a.emit_mov( + Size::S64, + Location::GPR(GPR::R10), + Location::Memory(GPR::RSP, 0), + ); + } + Location::XMM(_) => { + // Dummy value slot to be filled with `mov`. + a.emit_push(Size::S64, Location::GPR(GPR::RAX)); + + // XMM registers can be directly stored to memory. + a.emit_mov(Size::S64, *param, Location::Memory(GPR::RSP, 0)); + } + _ => a.emit_push(Size::S64, *param), + } + } + _ => { + return Err(CodegenError { + message: format!("emit_call_sysv loc: unreachable code"), + }) + } + } + } + + // Sort register moves so that register are not overwritten before read. + sort_call_movs(&mut call_movs); + + // Emit register moves. + for (loc, gpr) in call_movs { + if loc != Location::GPR(gpr) { + a.emit_mov(Size::S64, loc, Location::GPR(gpr)); + } + } + + // Put vmctx as the first parameter. + a.emit_mov( + Size::S64, + Location::GPR(Machine::get_vmctx_reg()), + Machine::get_param_location(0), + ); // vmctx + + if (m.state.stack_values.len() % 2) != 1 { + return Err(CodegenError { + message: format!("emit_call_sysv: explicit shadow takes one slot"), + }); + } + + cb(a); + + // Offset needs to be after the 'call' instruction. + if let Some((fsm, control_stack)) = state_context { + let state_diff_id = Self::get_state_diff(m, fsm, control_stack); + let offset = a.get_offset().0; + fsm.call_offsets.insert( + offset, + OffsetInfo { + end_offset: offset + 1, + activate_offset: offset, + diff_id: state_diff_id, + }, + ); + fsm.wasm_offset_to_target_offset + .insert(m.state.wasm_inst_offset, SuspendOffset::Call(offset)); + } + + // Restore stack. + if stack_offset > 0 { + a.emit_add( + Size::S64, + Location::Imm32(stack_offset as u32), + Location::GPR(GPR::RSP), + ); + if (stack_offset % 8) != 0 { + return Err(CodegenError { + message: format!("emit_call_sysv: Bad restoring stack alignement"), + }); + } + for _ in 0..stack_offset / 8 { + m.state.stack_values.pop().unwrap(); + } + } + + // Restore XMMs. + if used_xmms.len() > 0 { + for (i, r) in used_xmms.iter().enumerate() { + a.emit_mov( + Size::S64, + Location::Memory(GPR::RSP, (i * 8) as i32), + Location::XMM(*r), + ); + } + a.emit_add( + Size::S64, + Location::Imm32((used_xmms.len() * 8) as u32), + Location::GPR(GPR::RSP), + ); + for _ in 0..used_xmms.len() { + m.state.stack_values.pop().unwrap(); + } + } + + // Restore GPRs. + for r in used_gprs.iter().rev() { + a.emit_pop(Size::S64, Location::GPR(*r)); + m.state.stack_values.pop().unwrap(); + } + + if m.state.stack_values.pop().unwrap() != MachineValue::ExplicitShadow { + return Err(CodegenError { + message: format!("emit_call_sysv: Popped value is not ExplicitShadow"), + }); + } + Ok(()) + } + + /// Emits a System V call sequence, specialized for labels as the call target. + fn emit_call_sysv_label>( + a: &mut Assembler, + m: &mut Machine, + label: DynamicLabel, + params: I, + state_context: Option<(&mut FunctionStateMap, &mut [ControlFrame])>, + ) -> Result<(), CodegenError> { + Self::emit_call_sysv(a, m, |a| a.emit_call_label(label), params, state_context)?; + Ok(()) + } + + /// Emits a memory operation. + fn emit_memory_op Result<(), CodegenError>>( + module_info: &ModuleInfo, + config: &CodegenConfig, + a: &mut Assembler, + m: &mut Machine, + exception_table: &mut Option, + addr: Location, + memarg: &MemoryImmediate, + check_alignment: bool, + value_size: usize, + cb: F, + ) -> Result<(), CodegenError> { + // If the memory is dynamic, we need to do bound checking at runtime. + let mem_desc = match MemoryIndex::new(0).local_or_import(module_info) { + LocalOrImport::Local(local_mem_index) => &module_info.memories[local_mem_index], + LocalOrImport::Import(import_mem_index) => { + &module_info.imported_memories[import_mem_index].1 + } + }; + let need_check = match config.memory_bound_check_mode { + MemoryBoundCheckMode::Default => match mem_desc.memory_type() { + MemoryType::Dynamic => true, + MemoryType::Static | MemoryType::SharedStatic => false, + }, + MemoryBoundCheckMode::Enable => true, + MemoryBoundCheckMode::Disable => false, + }; + + let tmp_addr = m.acquire_temp_gpr().unwrap(); + let tmp_base = m.acquire_temp_gpr().unwrap(); + + // Load base into temporary register. + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + vm::Ctx::offset_memory_base() as i32, + ), + Location::GPR(tmp_base), + ); + + if need_check { + let tmp_bound = m.acquire_temp_gpr().unwrap(); + + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + vm::Ctx::offset_memory_bound() as i32, + ), + Location::GPR(tmp_bound), + ); + // Adds base to bound so `tmp_bound` now holds the end of linear memory. + a.emit_add(Size::S64, Location::GPR(tmp_base), Location::GPR(tmp_bound)); + a.emit_mov(Size::S32, addr, Location::GPR(tmp_addr)); + + // This branch is used for emitting "faster" code for the special case of (offset + value_size) not exceeding u32 range. + match (memarg.offset as u32).checked_add(value_size as u32) { + Some(0) => {} + Some(x) => { + a.emit_add(Size::S64, Location::Imm32(x), Location::GPR(tmp_addr)); + } + None => { + a.emit_add( + Size::S64, + Location::Imm32(memarg.offset as u32), + Location::GPR(tmp_addr), + ); + a.emit_add( + Size::S64, + Location::Imm32(value_size as u32), + Location::GPR(tmp_addr), + ); + } + } + + // Trap if the end address of the requested area is above that of the linear memory. + a.emit_add(Size::S64, Location::GPR(tmp_base), Location::GPR(tmp_addr)); + a.emit_cmp(Size::S64, Location::GPR(tmp_bound), Location::GPR(tmp_addr)); + + match exception_table { + Some(etable) => { + Self::mark_range_with_exception_code( + a, + etable, + ExceptionCode::MemoryOutOfBounds, + |a| a.emit_conditional_trap(Condition::Above), + ); + } + None => { a.emit_conditional_trap(Condition::Above); } + }; + + + m.release_temp_gpr(tmp_bound); + } + + // Calculates the real address, and loads from it. + a.emit_mov(Size::S32, addr, Location::GPR(tmp_addr)); + if memarg.offset != 0 { + a.emit_add( + Size::S64, + Location::Imm32(memarg.offset as u32), + Location::GPR(tmp_addr), + ); + } + a.emit_add(Size::S64, Location::GPR(tmp_base), Location::GPR(tmp_addr)); + m.release_temp_gpr(tmp_base); + + let align = match memarg.flags & 3 { + 0 => 1, + 1 => 2, + 2 => 4, + 3 => 8, + _ => { + return Err(CodegenError { + message: format!("emit_memory_op align: unreachable value"), + }) + } + }; + if check_alignment && align != 1 { + let tmp_aligncheck = m.acquire_temp_gpr().unwrap(); + a.emit_mov( + Size::S32, + Location::GPR(tmp_addr), + Location::GPR(tmp_aligncheck), + ); + a.emit_and( + Size::S64, + Location::Imm32(align - 1), + Location::GPR(tmp_aligncheck), + ); + match exception_table { + Some(etable) => { + Self::mark_range_with_exception_code( + a, + etable, + ExceptionCode::MemoryOutOfBounds, + |a| a.emit_conditional_trap(Condition::NotEqual), + ); + } + None => { a.emit_conditional_trap(Condition::NotEqual); } + }; + + + m.release_temp_gpr(tmp_aligncheck); + } + + match exception_table { + Some(etable) => { + Self::mark_range_with_exception_code(a, etable, ExceptionCode::MemoryOutOfBounds, |a| { + cb(a, m, tmp_addr) + })?; + } + None => { cb(a, m, tmp_addr)?; } + }; + + m.release_temp_gpr(tmp_addr); + Ok(()) + } + + /// Emits a memory operation. + fn emit_compare_and_swap( + module_info: &ModuleInfo, + config: &CodegenConfig, + a: &mut Assembler, + m: &mut Machine, + exception_table: &mut Option, + loc: Location, + target: Location, + ret: Location, + memarg: &MemoryImmediate, + value_size: usize, + memory_sz: Size, + stack_sz: Size, + cb: F, + ) -> Result<(), CodegenError> { + if memory_sz > stack_sz { + return Err(CodegenError { + message: format!("emit_compare_and_swap: memory size > stac size"), + }); + } + + let compare = m.reserve_unused_temp_gpr(GPR::RAX); + let value = if loc == Location::GPR(GPR::R14) { + GPR::R13 + } else { + GPR::R14 + }; + a.emit_push(Size::S64, Location::GPR(value)); + + a.emit_mov(stack_sz, loc, Location::GPR(value)); + + let retry = a.get_label(); + a.emit_label(retry); + + Self::emit_memory_op( + module_info, + config, + a, + m, + exception_table, + target, + memarg, + true, + value_size, + |a, m, addr| { + // Memory moves with size < 32b do not zero upper bits. + if memory_sz < Size::S32 { + a.emit_xor(Size::S32, Location::GPR(compare), Location::GPR(compare)); + } + a.emit_mov(memory_sz, Location::Memory(addr, 0), Location::GPR(compare)); + a.emit_mov(stack_sz, Location::GPR(compare), ret); + cb(a, m, compare, value); + a.emit_lock_cmpxchg(memory_sz, Location::GPR(value), Location::Memory(addr, 0)); + Ok(()) + }, + )?; + + a.emit_jmp(Condition::NotEqual, retry); + + a.emit_pop(Size::S64, Location::GPR(value)); + m.release_temp_gpr(compare); + Ok(()) + } + + // Checks for underflow/overflow/nan. + fn emit_f32_int_conv_check( + a: &mut Assembler, + m: &mut Machine, + reg: XMM, + lower_bound: f32, + upper_bound: f32, + underflow_label: ::Label, + overflow_label: ::Label, + nan_label: ::Label, + succeed_label: ::Label, + ) { + let lower_bound = f32::to_bits(lower_bound); + let upper_bound = f32::to_bits(upper_bound); + + let tmp = m.acquire_temp_gpr().unwrap(); + let tmp_x = m.acquire_temp_xmm().unwrap(); + + // Underflow. + a.emit_mov(Size::S32, Location::Imm32(lower_bound), Location::GPR(tmp)); + a.emit_mov(Size::S32, Location::GPR(tmp), Location::XMM(tmp_x)); + a.emit_vcmpless(reg, XMMOrMemory::XMM(tmp_x), tmp_x); + a.emit_mov(Size::S32, Location::XMM(tmp_x), Location::GPR(tmp)); + a.emit_cmp(Size::S32, Location::Imm32(0), Location::GPR(tmp)); + a.emit_jmp(Condition::NotEqual, underflow_label); + + // Overflow. + a.emit_mov(Size::S32, Location::Imm32(upper_bound), Location::GPR(tmp)); + a.emit_mov(Size::S32, Location::GPR(tmp), Location::XMM(tmp_x)); + a.emit_vcmpgess(reg, XMMOrMemory::XMM(tmp_x), tmp_x); + a.emit_mov(Size::S32, Location::XMM(tmp_x), Location::GPR(tmp)); + a.emit_cmp(Size::S32, Location::Imm32(0), Location::GPR(tmp)); + a.emit_jmp(Condition::NotEqual, overflow_label); + + // NaN. + a.emit_vcmpeqss(reg, XMMOrMemory::XMM(reg), tmp_x); + a.emit_mov(Size::S32, Location::XMM(tmp_x), Location::GPR(tmp)); + a.emit_cmp(Size::S32, Location::Imm32(0), Location::GPR(tmp)); + a.emit_jmp(Condition::Equal, nan_label); + + a.emit_jmp(Condition::None, succeed_label); + + m.release_temp_xmm(tmp_x); + m.release_temp_gpr(tmp); + } + + // Checks for underflow/overflow/nan before IxxTrunc{U/S}F32. + fn emit_f32_int_conv_check_trap( + a: &mut Assembler, + m: &mut Machine, + exception_table: &mut Option, + reg: XMM, + lower_bound: f32, + upper_bound: f32, + ) { + let trap = a.get_label(); + let end = a.get_label(); + + Self::emit_f32_int_conv_check(a, m, reg, lower_bound, upper_bound, trap, trap, trap, end); + a.emit_label(trap); + + match exception_table { + Some(etable) => { etable + .offset_to_code + .insert(a.get_offset().0, ExceptionCode::IllegalArithmetic); } + None => {} + }; + + a.emit_ud2(); + a.emit_label(end); + } + + fn emit_f32_int_conv_check_sat< + F1: FnOnce(&mut Assembler, &mut Machine), + F2: FnOnce(&mut Assembler, &mut Machine), + F3: FnOnce(&mut Assembler, &mut Machine), + F4: FnOnce(&mut Assembler, &mut Machine), + >( + a: &mut Assembler, + m: &mut Machine, + reg: XMM, + lower_bound: f32, + upper_bound: f32, + underflow_cb: F1, + overflow_cb: F2, + nan_cb: Option, + convert_cb: F4, + ) { + // As an optimization nan_cb is optional, and when set to None we turn + // use 'underflow' as the 'nan' label. This is useful for callers who + // set the return value to zero for both underflow and nan. + + let underflow = a.get_label(); + let overflow = a.get_label(); + let nan = if nan_cb.is_some() { + a.get_label() + } else { + underflow + }; + let convert = a.get_label(); + let end = a.get_label(); + + Self::emit_f32_int_conv_check( + a, + m, + reg, + lower_bound, + upper_bound, + underflow, + overflow, + nan, + convert, + ); + + a.emit_label(underflow); + underflow_cb(a, m); + a.emit_jmp(Condition::None, end); + + a.emit_label(overflow); + overflow_cb(a, m); + a.emit_jmp(Condition::None, end); + + if let Some(cb) = nan_cb { + a.emit_label(nan); + cb(a, m); + a.emit_jmp(Condition::None, end); + } + + a.emit_label(convert); + convert_cb(a, m); + a.emit_label(end); + } + + // Checks for underflow/overflow/nan. + fn emit_f64_int_conv_check( + a: &mut Assembler, + m: &mut Machine, + reg: XMM, + lower_bound: f64, + upper_bound: f64, + underflow_label: ::Label, + overflow_label: ::Label, + nan_label: ::Label, + succeed_label: ::Label, + ) { + let lower_bound = f64::to_bits(lower_bound); + let upper_bound = f64::to_bits(upper_bound); + + let tmp = m.acquire_temp_gpr().unwrap(); + let tmp_x = m.acquire_temp_xmm().unwrap(); + + // Underflow. + a.emit_mov(Size::S64, Location::Imm64(lower_bound), Location::GPR(tmp)); + a.emit_mov(Size::S64, Location::GPR(tmp), Location::XMM(tmp_x)); + a.emit_vcmplesd(reg, XMMOrMemory::XMM(tmp_x), tmp_x); + a.emit_mov(Size::S32, Location::XMM(tmp_x), Location::GPR(tmp)); + a.emit_cmp(Size::S32, Location::Imm32(0), Location::GPR(tmp)); + a.emit_jmp(Condition::NotEqual, underflow_label); + + // Overflow. + a.emit_mov(Size::S64, Location::Imm64(upper_bound), Location::GPR(tmp)); + a.emit_mov(Size::S64, Location::GPR(tmp), Location::XMM(tmp_x)); + a.emit_vcmpgesd(reg, XMMOrMemory::XMM(tmp_x), tmp_x); + a.emit_mov(Size::S32, Location::XMM(tmp_x), Location::GPR(tmp)); + a.emit_cmp(Size::S32, Location::Imm32(0), Location::GPR(tmp)); + a.emit_jmp(Condition::NotEqual, overflow_label); + + // NaN. + a.emit_vcmpeqsd(reg, XMMOrMemory::XMM(reg), tmp_x); + a.emit_mov(Size::S32, Location::XMM(tmp_x), Location::GPR(tmp)); + a.emit_cmp(Size::S32, Location::Imm32(0), Location::GPR(tmp)); + a.emit_jmp(Condition::Equal, nan_label); + + a.emit_jmp(Condition::None, succeed_label); + + m.release_temp_xmm(tmp_x); + m.release_temp_gpr(tmp); + } + + // Checks for underflow/overflow/nan before IxxTrunc{U/S}F64. + fn emit_f64_int_conv_check_trap( + a: &mut Assembler, + m: &mut Machine, + exception_table: &mut Option, + reg: XMM, + lower_bound: f64, + upper_bound: f64, + ) { + let trap = a.get_label(); + let end = a.get_label(); + + Self::emit_f64_int_conv_check(a, m, reg, lower_bound, upper_bound, trap, trap, trap, end); + a.emit_label(trap); + + match exception_table { + Some(etable) => { etable + .offset_to_code + .insert(a.get_offset().0, ExceptionCode::IllegalArithmetic); } + None => {} + }; + + a.emit_ud2(); + a.emit_label(end); + } + + fn emit_f64_int_conv_check_sat< + F1: FnOnce(&mut Assembler, &mut Machine), + F2: FnOnce(&mut Assembler, &mut Machine), + F3: FnOnce(&mut Assembler, &mut Machine), + F4: FnOnce(&mut Assembler, &mut Machine), + >( + a: &mut Assembler, + m: &mut Machine, + reg: XMM, + lower_bound: f64, + upper_bound: f64, + underflow_cb: F1, + overflow_cb: F2, + nan_cb: Option, + convert_cb: F4, + ) { + // As an optimization nan_cb is optional, and when set to None we turn + // use 'underflow' as the 'nan' label. This is useful for callers who + // set the return value to zero for both underflow and nan. + + let underflow = a.get_label(); + let overflow = a.get_label(); + let nan = if nan_cb.is_some() { + a.get_label() + } else { + underflow + }; + let convert = a.get_label(); + let end = a.get_label(); + + Self::emit_f64_int_conv_check( + a, + m, + reg, + lower_bound, + upper_bound, + underflow, + overflow, + nan, + convert, + ); + + a.emit_label(underflow); + underflow_cb(a, m); + a.emit_jmp(Condition::None, end); + + a.emit_label(overflow); + overflow_cb(a, m); + a.emit_jmp(Condition::None, end); + + if let Some(cb) = nan_cb { + a.emit_label(nan); + cb(a, m); + a.emit_jmp(Condition::None, end); + } + + a.emit_label(convert); + convert_cb(a, m); + a.emit_label(end); + } + + pub fn get_state_diff( + m: &Machine, + fsm: &mut FunctionStateMap, + control_stack: &mut [ControlFrame], + ) -> usize { + if !m.track_state { + return usize::MAX; + } + let last_frame = control_stack.last_mut().unwrap(); + let mut diff = m.state.diff(&last_frame.state); + diff.last = Some(last_frame.state_diff_id); + let id = fsm.diffs.len(); + last_frame.state = m.state.clone(); + last_frame.state_diff_id = id; + fsm.diffs.push(diff); + id + } +} + +impl FunctionCodeGenerator for X64FunctionCode { + fn feed_return(&mut self, ty: WpType) -> Result<(), CodegenError> { + self.returns.push(ty); + Ok(()) + } + + fn feed_param(&mut self, _ty: WpType) -> Result<(), CodegenError> { + self.num_params += 1; + self.num_locals += 1; + Ok(()) + } + + fn feed_local(&mut self, _ty: WpType, n: usize, _loc: u32) -> Result<(), CodegenError> { + self.num_locals += n; + Ok(()) + } + + fn begin_body(&mut self, _module_info: &ModuleInfo) -> Result<(), CodegenError> { + let a = self.assembler.as_mut().unwrap(); + let start_label = a.get_label(); + // skip the patchpoint during normal execution + a.emit_jmp(Condition::None, start_label); + // patchpoint of 32 1-byte nops + for _ in 0..32 { + a.emit_nop(); + } + a.emit_label(start_label); + a.emit_push(Size::S64, Location::GPR(GPR::RBP)); + a.emit_mov(Size::S64, Location::GPR(GPR::RSP), Location::GPR(GPR::RBP)); + + // Stack check. + if self.config.enforce_stack_check { + a.emit_cmp( + Size::S64, + Location::Memory( + GPR::RDI, // first parameter is vmctx + vm::Ctx::offset_stack_lower_bound() as i32, + ), + Location::GPR(GPR::RSP), + ); + match &mut self.exception_table { + Some(etable) => { + Self::mark_range_with_exception_code( + a, + etable, + ExceptionCode::MemoryOutOfBounds, + |a| a.emit_conditional_trap(Condition::Below), + ); + } + None => { a.emit_conditional_trap(Condition::Below); } + }; + } + + self.locals = self + .machine + .init_locals(a, self.num_locals, self.num_params); + + self.machine.state.register_values + [X64Register::GPR(Machine::get_vmctx_reg()).to_index().0] = MachineValue::Vmctx; + + self.fsm = FunctionStateMap::new( + new_machine_state(), + self.local_function_id, + 32, + (0..self.locals.len()) + .map(|_| WasmAbstractValue::Runtime) + .collect(), + ); + + let diff = self.machine.state.diff(&new_machine_state()); + let state_diff_id = self.fsm.diffs.len(); + self.fsm.diffs.push(diff); + + //println!("initial state = {:?}", self.machine.state); + + a.emit_sub(Size::S64, Location::Imm32(32), Location::GPR(GPR::RSP)); // simulate "red zone" if not supported by the platform + + self.control_stack.push(ControlFrame { + label: a.get_label(), + loop_like: false, + if_else: IfElseState::None, + returns: self.returns.clone(), + value_stack_depth: 0, + state: self.machine.state.clone(), + state_diff_id, + }); + + // Check interrupt signal without branching + let activate_offset = a.get_offset().0; + + if self.config.full_preemption { + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + vm::Ctx::offset_interrupt_signal_mem() as i32, + ), + Location::GPR(GPR::RAX), + ); + self.fsm.loop_offsets.insert( + a.get_offset().0, + OffsetInfo { + end_offset: a.get_offset().0 + 1, + activate_offset, + diff_id: state_diff_id, + }, + ); + self.fsm.wasm_function_header_target_offset = + Some(SuspendOffset::Loop(a.get_offset().0)); + a.emit_mov( + Size::S64, + Location::Memory(GPR::RAX, 0), + Location::GPR(GPR::RAX), + ); + } + + if self.machine.state.wasm_inst_offset != usize::MAX { + return Err(CodegenError { + message: format!("begin_body: wasm_inst_offset not usize::MAX"), + }); + } + Ok(()) + } + + fn finalize(&mut self) -> Result<(), CodegenError> { + let a = self.assembler.as_mut().unwrap(); + a.emit_ud2(); + Ok(()) + } + + fn feed_event( + &mut self, + ev: Event, + module_info: &ModuleInfo, + _source_loc: u32, + ) -> Result<(), CodegenError> { + let a = self.assembler.as_mut().unwrap(); + + match ev { + Event::Internal(InternalEvent::FunctionBegin(_)) + | Event::Internal(InternalEvent::FunctionEnd) => { + return Ok(()); + } + _ => {} + } + + self.machine.state.wasm_inst_offset = self.machine.state.wasm_inst_offset.wrapping_add(1); + + //println!("{:?} {}", op, self.value_stack.len()); + let was_unreachable; + + if self.unreachable_depth > 0 { + was_unreachable = true; + + if let Event::Wasm(op) = ev { + match *op { + Operator::Block { .. } | Operator::Loop { .. } | Operator::If { .. } => { + self.unreachable_depth += 1; + } + Operator::End => { + self.unreachable_depth -= 1; + } + Operator::Else => { + // We are in a reachable true branch + if self.unreachable_depth == 1 { + if let Some(IfElseState::If(_)) = + self.control_stack.last().map(|x| x.if_else) + { + self.unreachable_depth -= 1; + } + } + } + _ => {} + } + } + if self.unreachable_depth > 0 { + return Ok(()); + } + } else { + was_unreachable = false; + } + + let op = match ev { + Event::Wasm(x) => x, + Event::WasmOwned(ref x) => x, + Event::Internal(x) => { + match x { + InternalEvent::Breakpoint(callback) => { + self.breakpoints + .as_mut() + .unwrap() + .insert(a.get_offset(), callback); + Self::mark_trappable( + a, + &self.machine, + &mut self.fsm, + &mut self.control_stack, + ); + a.emit_inline_breakpoint(InlineBreakpointType::Middleware); + } + InternalEvent::FunctionBegin(_) | InternalEvent::FunctionEnd => {} + InternalEvent::GetInternal(idx) => { + let idx = idx as usize; + if idx >= INTERNALS_SIZE { + return Err(CodegenError { + message: format!("GetInternal: incorrect index value"), + }); + } + + let tmp = self.machine.acquire_temp_gpr().unwrap(); + + // Load `internals` pointer. + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + vm::Ctx::offset_internals() as i32, + ), + Location::GPR(tmp), + ); + + let loc = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(loc); + + // Move internal into the result location. + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + Location::Memory(tmp, (idx * 8) as i32), + loc, + ); + + self.machine.release_temp_gpr(tmp); + } + InternalEvent::SetInternal(idx) => { + let idx = idx as usize; + if idx >= INTERNALS_SIZE { + return Err(CodegenError { + message: format!("SetInternal: incorrect index value"), + }); + } + + let tmp = self.machine.acquire_temp_gpr().unwrap(); + + // Load `internals` pointer. + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + vm::Ctx::offset_internals() as i32, + ), + Location::GPR(tmp), + ); + let loc = get_location_released( + a, + &mut self.machine, + self.value_stack.pop().unwrap(), + ); + + // Move internal into storage. + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::Memory(tmp, (idx * 8) as i32), + ); + self.machine.release_temp_gpr(tmp); + } //_ => unimplemented!(), + } + return Ok(()); + } + }; + + match *op { + Operator::GlobalGet { global_index } => { + let global_index = global_index as usize; + + let tmp = self.machine.acquire_temp_gpr().unwrap(); + + let loc = match GlobalIndex::new(global_index).local_or_import(module_info) { + LocalOrImport::Local(local_index) => { + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + vm::Ctx::offset_globals() as i32, + ), + Location::GPR(tmp), + ); + a.emit_mov( + Size::S64, + Location::Memory(tmp, (local_index.index() as i32) * 8), + Location::GPR(tmp), + ); + self.machine.acquire_locations( + a, + &[( + type_to_wp_type(module_info.globals[local_index].desc.ty), + MachineValue::WasmStack(self.value_stack.len()), + )], + false, + )[0] + } + LocalOrImport::Import(import_index) => { + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + vm::Ctx::offset_imported_globals() as i32, + ), + Location::GPR(tmp), + ); + a.emit_mov( + Size::S64, + Location::Memory(tmp, (import_index.index() as i32) * 8), + Location::GPR(tmp), + ); + self.machine.acquire_locations( + a, + &[( + type_to_wp_type(module_info.imported_globals[import_index].1.ty), + MachineValue::WasmStack(self.value_stack.len()), + )], + false, + )[0] + } + }; + self.value_stack.push(loc); + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + Location::Memory(tmp, LocalGlobal::offset_data() as i32), + loc, + ); + + self.machine.release_temp_gpr(tmp); + } + Operator::GlobalSet { global_index } => { + let mut global_index = global_index as usize; + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + let tmp = self.machine.acquire_temp_gpr().unwrap(); + + if global_index < module_info.imported_globals.len() { + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + vm::Ctx::offset_imported_globals() as i32, + ), + Location::GPR(tmp), + ); + } else { + global_index -= module_info.imported_globals.len(); + if global_index >= module_info.globals.len() { + return Err(CodegenError { + message: format!("SetGlobal: incorrect global_index value"), + }); + } + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + vm::Ctx::offset_globals() as i32, + ), + Location::GPR(tmp), + ); + } + a.emit_mov( + Size::S64, + Location::Memory(tmp, (global_index as i32) * 8), + Location::GPR(tmp), + ); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::Memory(tmp, LocalGlobal::offset_data() as i32), + ); + + self.machine.release_temp_gpr(tmp); + } + Operator::LocalGet { local_index } => { + let local_index = local_index as usize; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + self.locals[local_index], + ret, + ); + self.value_stack.push(ret); + } + Operator::LocalSet { local_index } => { + let local_index = local_index as usize; + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + self.locals[local_index], + ); + } + Operator::LocalTee { local_index } => { + let local_index = local_index as usize; + let loc = *self.value_stack.last().unwrap(); + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + self.locals[local_index], + ); + } + Operator::I32Const { value } => { + self.value_stack.push(Location::Imm32(value as u32)); + self.machine + .state + .wasm_stack + .push(WasmAbstractValue::Const(value as u32 as u64)); + } + Operator::I32Add => Self::emit_binop_i32( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_add, + ), + Operator::I32Sub => Self::emit_binop_i32( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_sub, + ), + Operator::I32Mul => Self::emit_binop_i32( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_imul, + ), + Operator::I32DivU => { + // We assume that RAX and RDX are temporary registers here. + let loc_b = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let loc_a = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + a.emit_mov(Size::S32, loc_a, Location::GPR(GPR::RAX)); + a.emit_xor(Size::S32, Location::GPR(GPR::RDX), Location::GPR(GPR::RDX)); + Self::emit_relaxed_xdiv( + a, + &mut self.machine, + &mut self.exception_table, + Assembler::emit_div, + Size::S32, + loc_b, + &mut self.fsm, + &mut self.control_stack, + ); + a.emit_mov(Size::S32, Location::GPR(GPR::RAX), ret); + self.value_stack.push(ret); + } + Operator::I32DivS => { + // We assume that RAX and RDX are temporary registers here. + let loc_b = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let loc_a = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + a.emit_mov(Size::S32, loc_a, Location::GPR(GPR::RAX)); + a.emit_cdq(); + Self::emit_relaxed_xdiv( + a, + &mut self.machine, + &mut self.exception_table, + Assembler::emit_idiv, + Size::S32, + loc_b, + &mut self.fsm, + &mut self.control_stack, + ); + a.emit_mov(Size::S32, Location::GPR(GPR::RAX), ret); + self.value_stack.push(ret); + } + Operator::I32RemU => { + // We assume that RAX and RDX are temporary registers here. + let loc_b = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let loc_a = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + a.emit_mov(Size::S32, loc_a, Location::GPR(GPR::RAX)); + a.emit_xor(Size::S32, Location::GPR(GPR::RDX), Location::GPR(GPR::RDX)); + Self::emit_relaxed_xdiv( + a, + &mut self.machine, + &mut self.exception_table, + Assembler::emit_div, + Size::S32, + loc_b, + &mut self.fsm, + &mut self.control_stack, + ); + a.emit_mov(Size::S32, Location::GPR(GPR::RDX), ret); + self.value_stack.push(ret); + } + Operator::I32RemS => { + // We assume that RAX and RDX are temporary registers here. + let loc_b = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let loc_a = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + + let normal_path = a.get_label(); + let end = a.get_label(); + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_cmp, + Size::S32, + Location::Imm32(0x80000000), + loc_a, + ); + a.emit_jmp(Condition::NotEqual, normal_path); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_cmp, + Size::S32, + Location::Imm32(0xffffffff), + loc_b, + ); + a.emit_jmp(Condition::NotEqual, normal_path); + a.emit_mov(Size::S32, Location::Imm32(0), ret); + a.emit_jmp(Condition::None, end); + + a.emit_label(normal_path); + a.emit_mov(Size::S32, loc_a, Location::GPR(GPR::RAX)); + a.emit_cdq(); + Self::emit_relaxed_xdiv( + a, + &mut self.machine, + &mut self.exception_table, + Assembler::emit_idiv, + Size::S32, + loc_b, + &mut self.fsm, + &mut self.control_stack, + ); + a.emit_mov(Size::S32, Location::GPR(GPR::RDX), ret); + self.value_stack.push(ret); + + a.emit_label(end); + } + Operator::I32And => Self::emit_binop_i32( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_and, + ), + Operator::I32Or => Self::emit_binop_i32( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_or, + ), + Operator::I32Xor => Self::emit_binop_i32( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_xor, + ), + Operator::I32Eq => Self::emit_cmpop_i32( + a, + &mut self.machine, + &mut self.value_stack, + Condition::Equal, + )?, + Operator::I32Ne => Self::emit_cmpop_i32( + a, + &mut self.machine, + &mut self.value_stack, + Condition::NotEqual, + )?, + Operator::I32Eqz => Self::emit_cmpop_i32_dynamic_b( + a, + &mut self.machine, + &mut self.value_stack, + Condition::Equal, + Location::Imm32(0), + )?, + Operator::I32Clz => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let src = match loc { + Location::Imm32(_) | Location::Memory(_, _) => { + let tmp = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S32, loc, Location::GPR(tmp)); + tmp + } + Location::GPR(reg) => reg, + _ => { + return Err(CodegenError { + message: format!("I32Clz src: unreachable code"), + }) + } + }; + + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let dst = match ret { + Location::Memory(_, _) => self.machine.acquire_temp_gpr().unwrap(), + Location::GPR(reg) => reg, + _ => { + return Err(CodegenError { + message: format!("I32Clz dst: unreachable code"), + }) + } + }; + + if a.arch_has_xzcnt() { + a.arch_emit_lzcnt(Size::S32, Location::GPR(src), Location::GPR(dst)); + } else { + let zero_path = a.get_label(); + let end = a.get_label(); + + a.emit_test_gpr_64(src); + a.emit_jmp(Condition::Equal, zero_path); + a.emit_bsr(Size::S32, Location::GPR(src), Location::GPR(dst)); + a.emit_xor(Size::S32, Location::Imm32(31), Location::GPR(dst)); + a.emit_jmp(Condition::None, end); + a.emit_label(zero_path); + a.emit_mov(Size::S32, Location::Imm32(32), Location::GPR(dst)); + a.emit_label(end); + } + + match loc { + Location::Imm32(_) | Location::Memory(_, _) => { + self.machine.release_temp_gpr(src); + } + _ => {} + }; + match ret { + Location::Memory(_, _) => { + a.emit_mov(Size::S32, Location::GPR(dst), ret); + self.machine.release_temp_gpr(dst); + } + _ => {} + }; + } + Operator::I32Ctz => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let src = match loc { + Location::Imm32(_) | Location::Memory(_, _) => { + let tmp = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S32, loc, Location::GPR(tmp)); + tmp + } + Location::GPR(reg) => reg, + _ => { + return Err(CodegenError { + message: format!("I32Ctz src: unreachable code"), + }) + } + }; + + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let dst = match ret { + Location::Memory(_, _) => self.machine.acquire_temp_gpr().unwrap(), + Location::GPR(reg) => reg, + _ => { + return Err(CodegenError { + message: format!("I32Ctz dst: unreachable code"), + }) + } + }; + + if a.arch_has_xzcnt() { + a.arch_emit_tzcnt(Size::S32, Location::GPR(src), Location::GPR(dst)); + } else { + let zero_path = a.get_label(); + let end = a.get_label(); + + a.emit_test_gpr_64(src); + a.emit_jmp(Condition::Equal, zero_path); + a.emit_bsf(Size::S32, Location::GPR(src), Location::GPR(dst)); + a.emit_jmp(Condition::None, end); + a.emit_label(zero_path); + a.emit_mov(Size::S32, Location::Imm32(32), Location::GPR(dst)); + a.emit_label(end); + } + + match loc { + Location::Imm32(_) | Location::Memory(_, _) => { + self.machine.release_temp_gpr(src); + } + _ => {} + }; + match ret { + Location::Memory(_, _) => { + a.emit_mov(Size::S32, Location::GPR(dst), ret); + self.machine.release_temp_gpr(dst); + } + _ => {} + }; + } + Operator::I32Popcnt => Self::emit_xcnt_i32( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_popcnt, + )?, + Operator::I32Shl => Self::emit_shift_i32( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_shl, + ), + Operator::I32ShrU => Self::emit_shift_i32( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_shr, + ), + Operator::I32ShrS => Self::emit_shift_i32( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_sar, + ), + Operator::I32Rotl => Self::emit_shift_i32( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_rol, + ), + Operator::I32Rotr => Self::emit_shift_i32( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_ror, + ), + Operator::I32LtU => Self::emit_cmpop_i32( + a, + &mut self.machine, + &mut self.value_stack, + Condition::Below, + )?, + Operator::I32LeU => Self::emit_cmpop_i32( + a, + &mut self.machine, + &mut self.value_stack, + Condition::BelowEqual, + )?, + Operator::I32GtU => Self::emit_cmpop_i32( + a, + &mut self.machine, + &mut self.value_stack, + Condition::Above, + )?, + Operator::I32GeU => Self::emit_cmpop_i32( + a, + &mut self.machine, + &mut self.value_stack, + Condition::AboveEqual, + )?, + Operator::I32LtS => { + Self::emit_cmpop_i32(a, &mut self.machine, &mut self.value_stack, Condition::Less)?; + } + Operator::I32LeS => Self::emit_cmpop_i32( + a, + &mut self.machine, + &mut self.value_stack, + Condition::LessEqual, + )?, + Operator::I32GtS => Self::emit_cmpop_i32( + a, + &mut self.machine, + &mut self.value_stack, + Condition::Greater, + )?, + Operator::I32GeS => Self::emit_cmpop_i32( + a, + &mut self.machine, + &mut self.value_stack, + Condition::GreaterEqual, + )?, + Operator::I64Const { value } => { + let value = value as u64; + self.value_stack.push(Location::Imm64(value)); + self.machine + .state + .wasm_stack + .push(WasmAbstractValue::Const(value)); + } + Operator::I64Add => Self::emit_binop_i64( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_add, + ), + Operator::I64Sub => Self::emit_binop_i64( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_sub, + ), + Operator::I64Mul => Self::emit_binop_i64( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_imul, + ), + Operator::I64DivU => { + // We assume that RAX and RDX are temporary registers here. + let loc_b = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let loc_a = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + a.emit_mov(Size::S64, loc_a, Location::GPR(GPR::RAX)); + a.emit_xor(Size::S64, Location::GPR(GPR::RDX), Location::GPR(GPR::RDX)); + Self::emit_relaxed_xdiv( + a, + &mut self.machine, + &mut self.exception_table, + Assembler::emit_div, + Size::S64, + loc_b, + &mut self.fsm, + &mut self.control_stack, + ); + a.emit_mov(Size::S64, Location::GPR(GPR::RAX), ret); + self.value_stack.push(ret); + } + Operator::I64DivS => { + // We assume that RAX and RDX are temporary registers here. + let loc_b = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let loc_a = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + a.emit_mov(Size::S64, loc_a, Location::GPR(GPR::RAX)); + a.emit_cqo(); + Self::emit_relaxed_xdiv( + a, + &mut self.machine, + &mut self.exception_table, + Assembler::emit_idiv, + Size::S64, + loc_b, + &mut self.fsm, + &mut self.control_stack, + ); + a.emit_mov(Size::S64, Location::GPR(GPR::RAX), ret); + self.value_stack.push(ret); + } + Operator::I64RemU => { + // We assume that RAX and RDX are temporary registers here. + let loc_b = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let loc_a = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + a.emit_mov(Size::S64, loc_a, Location::GPR(GPR::RAX)); + a.emit_xor(Size::S64, Location::GPR(GPR::RDX), Location::GPR(GPR::RDX)); + Self::emit_relaxed_xdiv( + a, + &mut self.machine, + &mut self.exception_table, + Assembler::emit_div, + Size::S64, + loc_b, + &mut self.fsm, + &mut self.control_stack, + ); + a.emit_mov(Size::S64, Location::GPR(GPR::RDX), ret); + self.value_stack.push(ret); + } + Operator::I64RemS => { + // We assume that RAX and RDX are temporary registers here. + let loc_b = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let loc_a = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + + let normal_path = a.get_label(); + let end = a.get_label(); + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_cmp, + Size::S64, + Location::Imm64(0x8000000000000000u64), + loc_a, + ); + a.emit_jmp(Condition::NotEqual, normal_path); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_cmp, + Size::S64, + Location::Imm64(0xffffffffffffffffu64), + loc_b, + ); + a.emit_jmp(Condition::NotEqual, normal_path); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + Location::Imm64(0), + ret, + ); + a.emit_jmp(Condition::None, end); + + a.emit_label(normal_path); + + a.emit_mov(Size::S64, loc_a, Location::GPR(GPR::RAX)); + a.emit_cqo(); + Self::emit_relaxed_xdiv( + a, + &mut self.machine, + &mut self.exception_table, + Assembler::emit_idiv, + Size::S64, + loc_b, + &mut self.fsm, + &mut self.control_stack, + ); + a.emit_mov(Size::S64, Location::GPR(GPR::RDX), ret); + self.value_stack.push(ret); + a.emit_label(end); + } + Operator::I64And => Self::emit_binop_i64( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_and, + ), + Operator::I64Or => Self::emit_binop_i64( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_or, + ), + Operator::I64Xor => Self::emit_binop_i64( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_xor, + ), + Operator::I64Eq => Self::emit_cmpop_i64( + a, + &mut self.machine, + &mut self.value_stack, + Condition::Equal, + )?, + Operator::I64Ne => Self::emit_cmpop_i64( + a, + &mut self.machine, + &mut self.value_stack, + Condition::NotEqual, + )?, + Operator::I64Eqz => Self::emit_cmpop_i64_dynamic_b( + a, + &mut self.machine, + &mut self.value_stack, + Condition::Equal, + Location::Imm64(0), + )?, + Operator::I64Clz => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let src = match loc { + Location::Imm64(_) | Location::Imm32(_) | Location::Memory(_, _) => { + let tmp = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S64, loc, Location::GPR(tmp)); + tmp + } + Location::GPR(reg) => reg, + _ => { + return Err(CodegenError { + message: format!("I64Clz src: unreachable code"), + }) + } + }; + + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let dst = match ret { + Location::Memory(_, _) => self.machine.acquire_temp_gpr().unwrap(), + Location::GPR(reg) => reg, + _ => { + return Err(CodegenError { + message: format!("I64Clz dst: unreachable code"), + }) + } + }; + + if a.arch_has_xzcnt() { + a.arch_emit_lzcnt(Size::S64, Location::GPR(src), Location::GPR(dst)); + } else { + let zero_path = a.get_label(); + let end = a.get_label(); + + a.emit_test_gpr_64(src); + a.emit_jmp(Condition::Equal, zero_path); + a.emit_bsr(Size::S64, Location::GPR(src), Location::GPR(dst)); + a.emit_xor(Size::S64, Location::Imm32(63), Location::GPR(dst)); + a.emit_jmp(Condition::None, end); + a.emit_label(zero_path); + a.emit_mov(Size::S64, Location::Imm32(64), Location::GPR(dst)); + a.emit_label(end); + } + + match loc { + Location::Imm64(_) | Location::Imm32(_) | Location::Memory(_, _) => { + self.machine.release_temp_gpr(src); + } + _ => {} + }; + match ret { + Location::Memory(_, _) => { + a.emit_mov(Size::S64, Location::GPR(dst), ret); + self.machine.release_temp_gpr(dst); + } + _ => {} + }; + } + Operator::I64Ctz => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let src = match loc { + Location::Imm64(_) | Location::Imm32(_) | Location::Memory(_, _) => { + let tmp = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S64, loc, Location::GPR(tmp)); + tmp + } + Location::GPR(reg) => reg, + _ => { + return Err(CodegenError { + message: format!("I64Ctz src: unreachable code"), + }) + } + }; + + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let dst = match ret { + Location::Memory(_, _) => self.machine.acquire_temp_gpr().unwrap(), + Location::GPR(reg) => reg, + _ => { + return Err(CodegenError { + message: format!("I64Ctz dst: unreachable code"), + }) + } + }; + + if a.arch_has_xzcnt() { + a.arch_emit_tzcnt(Size::S64, Location::GPR(src), Location::GPR(dst)); + } else { + let zero_path = a.get_label(); + let end = a.get_label(); + + a.emit_test_gpr_64(src); + a.emit_jmp(Condition::Equal, zero_path); + a.emit_bsf(Size::S64, Location::GPR(src), Location::GPR(dst)); + a.emit_jmp(Condition::None, end); + a.emit_label(zero_path); + a.emit_mov(Size::S64, Location::Imm32(64), Location::GPR(dst)); + a.emit_label(end); + } + + match loc { + Location::Imm64(_) | Location::Imm32(_) | Location::Memory(_, _) => { + self.machine.release_temp_gpr(src); + } + _ => {} + }; + match ret { + Location::Memory(_, _) => { + a.emit_mov(Size::S64, Location::GPR(dst), ret); + self.machine.release_temp_gpr(dst); + } + _ => {} + }; + } + Operator::I64Popcnt => Self::emit_xcnt_i64( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_popcnt, + )?, + Operator::I64Shl => Self::emit_shift_i64( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_shl, + ), + Operator::I64ShrU => Self::emit_shift_i64( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_shr, + ), + Operator::I64ShrS => Self::emit_shift_i64( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_sar, + ), + Operator::I64Rotl => Self::emit_shift_i64( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_rol, + ), + Operator::I64Rotr => Self::emit_shift_i64( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_ror, + ), + Operator::I64LtU => Self::emit_cmpop_i64( + a, + &mut self.machine, + &mut self.value_stack, + Condition::Below, + )?, + Operator::I64LeU => Self::emit_cmpop_i64( + a, + &mut self.machine, + &mut self.value_stack, + Condition::BelowEqual, + )?, + Operator::I64GtU => Self::emit_cmpop_i64( + a, + &mut self.machine, + &mut self.value_stack, + Condition::Above, + )?, + Operator::I64GeU => Self::emit_cmpop_i64( + a, + &mut self.machine, + &mut self.value_stack, + Condition::AboveEqual, + )?, + Operator::I64LtS => { + Self::emit_cmpop_i64(a, &mut self.machine, &mut self.value_stack, Condition::Less)?; + } + Operator::I64LeS => Self::emit_cmpop_i64( + a, + &mut self.machine, + &mut self.value_stack, + Condition::LessEqual, + )?, + Operator::I64GtS => Self::emit_cmpop_i64( + a, + &mut self.machine, + &mut self.value_stack, + Condition::Greater, + )?, + Operator::I64GeS => Self::emit_cmpop_i64( + a, + &mut self.machine, + &mut self.value_stack, + Condition::GreaterEqual, + )?, + Operator::I64ExtendI32U => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + ret, + ); + } + Operator::I64ExtendI32S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + Self::emit_relaxed_zx_sx( + a, + &mut self.machine, + Assembler::emit_movsx, + Size::S32, + loc, + Size::S64, + ret, + )?; + } + Operator::I32Extend8S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_relaxed_zx_sx( + a, + &mut self.machine, + Assembler::emit_movsx, + Size::S8, + loc, + Size::S32, + ret, + )?; + } + Operator::I32Extend16S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_relaxed_zx_sx( + a, + &mut self.machine, + Assembler::emit_movsx, + Size::S16, + loc, + Size::S32, + ret, + )?; + } + Operator::I64Extend8S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_relaxed_zx_sx( + a, + &mut self.machine, + Assembler::emit_movsx, + Size::S8, + loc, + Size::S64, + ret, + )?; + } + Operator::I64Extend16S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_relaxed_zx_sx( + a, + &mut self.machine, + Assembler::emit_movsx, + Size::S16, + loc, + Size::S64, + ret, + )?; + } + Operator::I64Extend32S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_relaxed_zx_sx( + a, + &mut self.machine, + Assembler::emit_movsx, + Size::S32, + loc, + Size::S64, + ret, + )?; + } + Operator::I32WrapI64 => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + ret, + ); + } + + Operator::F32Const { value } => { + self.value_stack.push(Location::Imm32(value.bits())); + self.machine + .state + .wasm_stack + .push(WasmAbstractValue::Const(value.bits() as u64)); + } + Operator::F32Add => Self::emit_fp_binop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vaddss, + )?, + Operator::F32Sub => Self::emit_fp_binop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vsubss, + )?, + Operator::F32Mul => Self::emit_fp_binop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vmulss, + )?, + Operator::F32Div => Self::emit_fp_binop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vdivss, + )?, + Operator::F32Max => { + if !a.arch_supports_canonicalize_nan() { + Self::emit_fp_binop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vmaxss, + )?; + } else { + let src2 = get_location_released( + a, + &mut self.machine, + self.value_stack.pop().unwrap(), + ); + let src1 = get_location_released( + a, + &mut self.machine, + self.value_stack.pop().unwrap(), + ); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let tmp1 = self.machine.acquire_temp_xmm().unwrap(); + let tmp2 = self.machine.acquire_temp_xmm().unwrap(); + let tmpg1 = self.machine.acquire_temp_gpr().unwrap(); + let tmpg2 = self.machine.acquire_temp_gpr().unwrap(); + + let src1 = match src1 { + Location::XMM(x) => x, + Location::GPR(_) | Location::Memory(_, _) => { + a.emit_mov(Size::S64, src1, Location::XMM(tmp1)); + tmp1 + } + Location::Imm32(_) => { + a.emit_mov(Size::S32, src1, Location::GPR(tmpg1)); + a.emit_mov(Size::S32, Location::GPR(tmpg1), Location::XMM(tmp1)); + tmp1 + } + Location::Imm64(_) => { + a.emit_mov(Size::S64, src1, Location::GPR(tmpg1)); + a.emit_mov(Size::S64, Location::GPR(tmpg1), Location::XMM(tmp1)); + tmp1 + } + _ => { + return Err(CodegenError { + message: format!("F32Max src1: unreachable code"), + }) + } + }; + let src2 = match src2 { + Location::XMM(x) => x, + Location::GPR(_) | Location::Memory(_, _) => { + a.emit_mov(Size::S64, src2, Location::XMM(tmp2)); + tmp2 + } + Location::Imm32(_) => { + a.emit_mov(Size::S32, src2, Location::GPR(tmpg1)); + a.emit_mov(Size::S32, Location::GPR(tmpg1), Location::XMM(tmp2)); + tmp2 + } + Location::Imm64(_) => { + a.emit_mov(Size::S64, src2, Location::GPR(tmpg1)); + a.emit_mov(Size::S64, Location::GPR(tmpg1), Location::XMM(tmp2)); + tmp2 + } + _ => { + return Err(CodegenError { + message: format!("F32Max src2: unreachable code"), + }) + } + }; + + let tmp_xmm1 = XMM::XMM8; + let tmp_xmm2 = XMM::XMM9; + let tmp_xmm3 = XMM::XMM10; + + a.emit_mov(Size::S32, Location::XMM(src1), Location::GPR(tmpg1)); + a.emit_mov(Size::S32, Location::XMM(src2), Location::GPR(tmpg2)); + a.emit_cmp(Size::S32, Location::GPR(tmpg2), Location::GPR(tmpg1)); + a.emit_vmaxss(src1, XMMOrMemory::XMM(src2), tmp_xmm1); + let label1 = a.get_label(); + let label2 = a.get_label(); + a.emit_jmp(Condition::NotEqual, label1); + a.emit_vmovaps(XMMOrMemory::XMM(tmp_xmm1), XMMOrMemory::XMM(tmp_xmm2)); + a.emit_jmp(Condition::None, label2); + a.emit_label(label1); + a.emit_vxorps(tmp_xmm2, XMMOrMemory::XMM(tmp_xmm2), tmp_xmm2); + a.emit_label(label2); + a.emit_vcmpeqss(src1, XMMOrMemory::XMM(src2), tmp_xmm3); + a.emit_vblendvps(tmp_xmm3, XMMOrMemory::XMM(tmp_xmm2), tmp_xmm1, tmp_xmm1); + a.emit_vcmpunordss(src1, XMMOrMemory::XMM(src2), src1); + // load float canonical nan + a.emit_mov( + Size::S64, + Location::Imm32(0x7FC0_0000), // Canonical NaN + Location::GPR(tmpg1), + ); + a.emit_mov(Size::S64, Location::GPR(tmpg1), Location::XMM(src2)); + a.emit_vblendvps(src1, XMMOrMemory::XMM(src2), tmp_xmm1, src1); + match ret { + Location::XMM(x) => { + a.emit_vmovaps(XMMOrMemory::XMM(src1), XMMOrMemory::XMM(x)); + } + Location::Memory(_, _) | Location::GPR(_) => { + a.emit_mov(Size::S64, Location::XMM(src1), ret); + } + _ => { + return Err(CodegenError { + message: format!("F32Max ret: unreachable code"), + }) + } + } + + self.machine.release_temp_gpr(tmpg2); + self.machine.release_temp_gpr(tmpg1); + self.machine.release_temp_xmm(tmp2); + self.machine.release_temp_xmm(tmp1); + } + } + Operator::F32Min => { + if !a.arch_supports_canonicalize_nan() { + Self::emit_fp_binop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vminss, + )?; + } else { + let src2 = get_location_released( + a, + &mut self.machine, + self.value_stack.pop().unwrap(), + ); + let src1 = get_location_released( + a, + &mut self.machine, + self.value_stack.pop().unwrap(), + ); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let tmp1 = self.machine.acquire_temp_xmm().unwrap(); + let tmp2 = self.machine.acquire_temp_xmm().unwrap(); + let tmpg1 = self.machine.acquire_temp_gpr().unwrap(); + let tmpg2 = self.machine.acquire_temp_gpr().unwrap(); + + let src1 = match src1 { + Location::XMM(x) => x, + Location::GPR(_) | Location::Memory(_, _) => { + a.emit_mov(Size::S64, src1, Location::XMM(tmp1)); + tmp1 + } + Location::Imm32(_) => { + a.emit_mov(Size::S32, src1, Location::GPR(tmpg1)); + a.emit_mov(Size::S32, Location::GPR(tmpg1), Location::XMM(tmp1)); + tmp1 + } + Location::Imm64(_) => { + a.emit_mov(Size::S64, src1, Location::GPR(tmpg1)); + a.emit_mov(Size::S64, Location::GPR(tmpg1), Location::XMM(tmp1)); + tmp1 + } + _ => { + return Err(CodegenError { + message: format!("F32Min src1: unreachable code"), + }) + } + }; + let src2 = match src2 { + Location::XMM(x) => x, + Location::GPR(_) | Location::Memory(_, _) => { + a.emit_mov(Size::S64, src2, Location::XMM(tmp2)); + tmp2 + } + Location::Imm32(_) => { + a.emit_mov(Size::S32, src2, Location::GPR(tmpg1)); + a.emit_mov(Size::S32, Location::GPR(tmpg1), Location::XMM(tmp2)); + tmp2 + } + Location::Imm64(_) => { + a.emit_mov(Size::S64, src2, Location::GPR(tmpg1)); + a.emit_mov(Size::S64, Location::GPR(tmpg1), Location::XMM(tmp2)); + tmp2 + } + _ => { + return Err(CodegenError { + message: format!("F32Min src2: unreachable code"), + }) + } + }; + + let tmp_xmm1 = XMM::XMM8; + let tmp_xmm2 = XMM::XMM9; + let tmp_xmm3 = XMM::XMM10; + + a.emit_mov(Size::S32, Location::XMM(src1), Location::GPR(tmpg1)); + a.emit_mov(Size::S32, Location::XMM(src2), Location::GPR(tmpg2)); + a.emit_cmp(Size::S32, Location::GPR(tmpg2), Location::GPR(tmpg1)); + a.emit_vminss(src1, XMMOrMemory::XMM(src2), tmp_xmm1); + let label1 = a.get_label(); + let label2 = a.get_label(); + a.emit_jmp(Condition::NotEqual, label1); + a.emit_vmovaps(XMMOrMemory::XMM(tmp_xmm1), XMMOrMemory::XMM(tmp_xmm2)); + a.emit_jmp(Condition::None, label2); + a.emit_label(label1); + // load float -0.0 + a.emit_mov( + Size::S64, + Location::Imm32(0x8000_0000), // Negative zero + Location::GPR(tmpg1), + ); + a.emit_mov(Size::S64, Location::GPR(tmpg1), Location::XMM(tmp_xmm2)); + a.emit_label(label2); + a.emit_vcmpeqss(src1, XMMOrMemory::XMM(src2), tmp_xmm3); + a.emit_vblendvps(tmp_xmm3, XMMOrMemory::XMM(tmp_xmm2), tmp_xmm1, tmp_xmm1); + a.emit_vcmpunordss(src1, XMMOrMemory::XMM(src2), src1); + // load float canonical nan + a.emit_mov( + Size::S64, + Location::Imm32(0x7FC0_0000), // Canonical NaN + Location::GPR(tmpg1), + ); + a.emit_mov(Size::S64, Location::GPR(tmpg1), Location::XMM(src2)); + a.emit_vblendvps(src1, XMMOrMemory::XMM(src2), tmp_xmm1, src1); + match ret { + Location::XMM(x) => { + a.emit_vmovaps(XMMOrMemory::XMM(src1), XMMOrMemory::XMM(x)); + } + Location::Memory(_, _) | Location::GPR(_) => { + a.emit_mov(Size::S64, Location::XMM(src1), ret); + } + _ => { + return Err(CodegenError { + message: format!("F32Min ret: unreachable code"), + }) + } + } + + self.machine.release_temp_gpr(tmpg2); + self.machine.release_temp_gpr(tmpg1); + self.machine.release_temp_xmm(tmp2); + self.machine.release_temp_xmm(tmp1); + } + } + Operator::F32Eq => Self::emit_fp_cmpop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vcmpeqss, + )?, + Operator::F32Ne => Self::emit_fp_cmpop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vcmpneqss, + )?, + Operator::F32Lt => Self::emit_fp_cmpop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vcmpltss, + )?, + Operator::F32Le => Self::emit_fp_cmpop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vcmpless, + )?, + Operator::F32Gt => Self::emit_fp_cmpop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vcmpgtss, + )?, + Operator::F32Ge => Self::emit_fp_cmpop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vcmpgess, + )?, + Operator::F32Nearest => Self::emit_fp_unop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vroundss_nearest, + )?, + Operator::F32Floor => Self::emit_fp_unop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vroundss_floor, + )?, + Operator::F32Ceil => Self::emit_fp_unop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vroundss_ceil, + )?, + Operator::F32Trunc => Self::emit_fp_unop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vroundss_trunc, + )?, + Operator::F32Sqrt => Self::emit_fp_unop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vsqrtss, + )?, + + Operator::F32Copysign => { + let loc_b = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let loc_a = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let tmp1 = self.machine.acquire_temp_gpr().unwrap(); + let tmp2 = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S32, loc_a, Location::GPR(tmp1)); + a.emit_mov(Size::S32, loc_b, Location::GPR(tmp2)); + a.emit_and( + Size::S32, + Location::Imm32(0x7fffffffu32), + Location::GPR(tmp1), + ); + a.emit_and( + Size::S32, + Location::Imm32(0x80000000u32), + Location::GPR(tmp2), + ); + a.emit_or(Size::S32, Location::GPR(tmp2), Location::GPR(tmp1)); + a.emit_mov(Size::S32, Location::GPR(tmp1), ret); + self.machine.release_temp_gpr(tmp2); + self.machine.release_temp_gpr(tmp1); + } + + Operator::F32Abs => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + let tmp = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S32, loc, Location::GPR(tmp)); + a.emit_and( + Size::S32, + Location::Imm32(0x7fffffffu32), + Location::GPR(tmp), + ); + a.emit_mov(Size::S32, Location::GPR(tmp), ret); + self.machine.release_temp_gpr(tmp); + } + + Operator::F32Neg => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + if a.arch_has_fneg() { + let tmp = self.machine.acquire_temp_xmm().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::XMM(tmp), + ); + a.arch_emit_f32_neg(tmp, tmp); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + Location::XMM(tmp), + ret, + ); + self.machine.release_temp_xmm(tmp); + } else { + let tmp = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S32, loc, Location::GPR(tmp)); + a.emit_btc_gpr_imm8_32(31, tmp); + a.emit_mov(Size::S32, Location::GPR(tmp), ret); + self.machine.release_temp_gpr(tmp); + } + } + + Operator::F64Const { value } => { + self.value_stack.push(Location::Imm64(value.bits())); + self.machine + .state + .wasm_stack + .push(WasmAbstractValue::Const(value.bits())); + } + Operator::F64Add => Self::emit_fp_binop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vaddsd, + )?, + Operator::F64Sub => Self::emit_fp_binop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vsubsd, + )?, + Operator::F64Mul => Self::emit_fp_binop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vmulsd, + )?, + Operator::F64Div => Self::emit_fp_binop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vdivsd, + )?, + Operator::F64Max => { + if !a.arch_supports_canonicalize_nan() { + Self::emit_fp_binop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vmaxsd, + )?; + } else { + let src2 = get_location_released( + a, + &mut self.machine, + self.value_stack.pop().unwrap(), + ); + let src1 = get_location_released( + a, + &mut self.machine, + self.value_stack.pop().unwrap(), + ); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let tmp1 = self.machine.acquire_temp_xmm().unwrap(); + let tmp2 = self.machine.acquire_temp_xmm().unwrap(); + let tmpg1 = self.machine.acquire_temp_gpr().unwrap(); + let tmpg2 = self.machine.acquire_temp_gpr().unwrap(); + + let src1 = match src1 { + Location::XMM(x) => x, + Location::GPR(_) | Location::Memory(_, _) => { + a.emit_mov(Size::S64, src1, Location::XMM(tmp1)); + tmp1 + } + Location::Imm32(_) => { + a.emit_mov(Size::S32, src1, Location::GPR(tmpg1)); + a.emit_mov(Size::S32, Location::GPR(tmpg1), Location::XMM(tmp1)); + tmp1 + } + Location::Imm64(_) => { + a.emit_mov(Size::S64, src1, Location::GPR(tmpg1)); + a.emit_mov(Size::S64, Location::GPR(tmpg1), Location::XMM(tmp1)); + tmp1 + } + _ => { + return Err(CodegenError { + message: format!("F64Max src1: unreachable code"), + }) + } + }; + let src2 = match src2 { + Location::XMM(x) => x, + Location::GPR(_) | Location::Memory(_, _) => { + a.emit_mov(Size::S64, src2, Location::XMM(tmp2)); + tmp2 + } + Location::Imm32(_) => { + a.emit_mov(Size::S32, src2, Location::GPR(tmpg1)); + a.emit_mov(Size::S32, Location::GPR(tmpg1), Location::XMM(tmp2)); + tmp2 + } + Location::Imm64(_) => { + a.emit_mov(Size::S64, src2, Location::GPR(tmpg1)); + a.emit_mov(Size::S64, Location::GPR(tmpg1), Location::XMM(tmp2)); + tmp2 + } + _ => { + return Err(CodegenError { + message: format!("F64Max src2: unreachable code"), + }) + } + }; + + let tmp_xmm1 = XMM::XMM8; + let tmp_xmm2 = XMM::XMM9; + let tmp_xmm3 = XMM::XMM10; + + a.emit_mov(Size::S64, Location::XMM(src1), Location::GPR(tmpg1)); + a.emit_mov(Size::S64, Location::XMM(src2), Location::GPR(tmpg2)); + a.emit_cmp(Size::S64, Location::GPR(tmpg2), Location::GPR(tmpg1)); + a.emit_vmaxsd(src1, XMMOrMemory::XMM(src2), tmp_xmm1); + let label1 = a.get_label(); + let label2 = a.get_label(); + a.emit_jmp(Condition::NotEqual, label1); + a.emit_vmovapd(XMMOrMemory::XMM(tmp_xmm1), XMMOrMemory::XMM(tmp_xmm2)); + a.emit_jmp(Condition::None, label2); + a.emit_label(label1); + a.emit_vxorpd(tmp_xmm2, XMMOrMemory::XMM(tmp_xmm2), tmp_xmm2); + a.emit_label(label2); + a.emit_vcmpeqsd(src1, XMMOrMemory::XMM(src2), tmp_xmm3); + a.emit_vblendvpd(tmp_xmm3, XMMOrMemory::XMM(tmp_xmm2), tmp_xmm1, tmp_xmm1); + a.emit_vcmpunordsd(src1, XMMOrMemory::XMM(src2), src1); + // load float canonical nan + a.emit_mov( + Size::S64, + Location::Imm64(0x7FF8_0000_0000_0000), // Canonical NaN + Location::GPR(tmpg1), + ); + a.emit_mov(Size::S64, Location::GPR(tmpg1), Location::XMM(src2)); + a.emit_vblendvpd(src1, XMMOrMemory::XMM(src2), tmp_xmm1, src1); + match ret { + Location::XMM(x) => { + a.emit_vmovapd(XMMOrMemory::XMM(src1), XMMOrMemory::XMM(x)); + } + Location::Memory(_, _) | Location::GPR(_) => { + a.emit_mov(Size::S64, Location::XMM(src1), ret); + } + _ => { + return Err(CodegenError { + message: format!("F64Max ret: unreachable code"), + }) + } + } + + self.machine.release_temp_gpr(tmpg2); + self.machine.release_temp_gpr(tmpg1); + self.machine.release_temp_xmm(tmp2); + self.machine.release_temp_xmm(tmp1); + } + } + Operator::F64Min => { + if !a.arch_supports_canonicalize_nan() { + Self::emit_fp_binop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vminsd, + )?; + } else { + let src2 = get_location_released( + a, + &mut self.machine, + self.value_stack.pop().unwrap(), + ); + let src1 = get_location_released( + a, + &mut self.machine, + self.value_stack.pop().unwrap(), + ); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let tmp1 = self.machine.acquire_temp_xmm().unwrap(); + let tmp2 = self.machine.acquire_temp_xmm().unwrap(); + let tmpg1 = self.machine.acquire_temp_gpr().unwrap(); + let tmpg2 = self.machine.acquire_temp_gpr().unwrap(); + + let src1 = match src1 { + Location::XMM(x) => x, + Location::GPR(_) | Location::Memory(_, _) => { + a.emit_mov(Size::S64, src1, Location::XMM(tmp1)); + tmp1 + } + Location::Imm32(_) => { + a.emit_mov(Size::S32, src1, Location::GPR(tmpg1)); + a.emit_mov(Size::S32, Location::GPR(tmpg1), Location::XMM(tmp1)); + tmp1 + } + Location::Imm64(_) => { + a.emit_mov(Size::S64, src1, Location::GPR(tmpg1)); + a.emit_mov(Size::S64, Location::GPR(tmpg1), Location::XMM(tmp1)); + tmp1 + } + _ => { + return Err(CodegenError { + message: format!("F64Min src1: unreachable code"), + }) + } + }; + let src2 = match src2 { + Location::XMM(x) => x, + Location::GPR(_) | Location::Memory(_, _) => { + a.emit_mov(Size::S64, src2, Location::XMM(tmp2)); + tmp2 + } + Location::Imm32(_) => { + a.emit_mov(Size::S32, src2, Location::GPR(tmpg1)); + a.emit_mov(Size::S32, Location::GPR(tmpg1), Location::XMM(tmp2)); + tmp2 + } + Location::Imm64(_) => { + a.emit_mov(Size::S64, src2, Location::GPR(tmpg1)); + a.emit_mov(Size::S64, Location::GPR(tmpg1), Location::XMM(tmp2)); + tmp2 + } + _ => { + return Err(CodegenError { + message: format!("F64Min src2: unreachable code"), + }) + } + }; + + let tmp_xmm1 = XMM::XMM8; + let tmp_xmm2 = XMM::XMM9; + let tmp_xmm3 = XMM::XMM10; + + a.emit_mov(Size::S64, Location::XMM(src1), Location::GPR(tmpg1)); + a.emit_mov(Size::S64, Location::XMM(src2), Location::GPR(tmpg2)); + a.emit_cmp(Size::S64, Location::GPR(tmpg2), Location::GPR(tmpg1)); + a.emit_vminsd(src1, XMMOrMemory::XMM(src2), tmp_xmm1); + let label1 = a.get_label(); + let label2 = a.get_label(); + a.emit_jmp(Condition::NotEqual, label1); + a.emit_vmovapd(XMMOrMemory::XMM(tmp_xmm1), XMMOrMemory::XMM(tmp_xmm2)); + a.emit_jmp(Condition::None, label2); + a.emit_label(label1); + // load float -0.0 + a.emit_mov( + Size::S64, + Location::Imm64(0x8000_0000_0000_0000), // Negative zero + Location::GPR(tmpg1), + ); + a.emit_mov(Size::S64, Location::GPR(tmpg1), Location::XMM(tmp_xmm2)); + a.emit_label(label2); + a.emit_vcmpeqsd(src1, XMMOrMemory::XMM(src2), tmp_xmm3); + a.emit_vblendvpd(tmp_xmm3, XMMOrMemory::XMM(tmp_xmm2), tmp_xmm1, tmp_xmm1); + a.emit_vcmpunordsd(src1, XMMOrMemory::XMM(src2), src1); + // load float canonical nan + a.emit_mov( + Size::S64, + Location::Imm64(0x7FF8_0000_0000_0000), // Canonical NaN + Location::GPR(tmpg1), + ); + a.emit_mov(Size::S64, Location::GPR(tmpg1), Location::XMM(src2)); + a.emit_vblendvpd(src1, XMMOrMemory::XMM(src2), tmp_xmm1, src1); + match ret { + Location::XMM(x) => { + a.emit_vmovaps(XMMOrMemory::XMM(src1), XMMOrMemory::XMM(x)); + } + Location::Memory(_, _) | Location::GPR(_) => { + a.emit_mov(Size::S64, Location::XMM(src1), ret); + } + _ => { + return Err(CodegenError { + message: format!("F64Min ret: unreachable code"), + }) + } + } + + self.machine.release_temp_gpr(tmpg2); + self.machine.release_temp_gpr(tmpg1); + self.machine.release_temp_xmm(tmp2); + self.machine.release_temp_xmm(tmp1); + } + } + Operator::F64Eq => Self::emit_fp_cmpop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vcmpeqsd, + )?, + Operator::F64Ne => Self::emit_fp_cmpop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vcmpneqsd, + )?, + Operator::F64Lt => Self::emit_fp_cmpop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vcmpltsd, + )?, + Operator::F64Le => Self::emit_fp_cmpop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vcmplesd, + )?, + Operator::F64Gt => Self::emit_fp_cmpop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vcmpgtsd, + )?, + Operator::F64Ge => Self::emit_fp_cmpop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vcmpgesd, + )?, + Operator::F64Nearest => Self::emit_fp_unop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vroundsd_nearest, + )?, + Operator::F64Floor => Self::emit_fp_unop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vroundsd_floor, + )?, + Operator::F64Ceil => Self::emit_fp_unop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vroundsd_ceil, + )?, + Operator::F64Trunc => Self::emit_fp_unop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vroundsd_trunc, + )?, + Operator::F64Sqrt => Self::emit_fp_unop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vsqrtsd, + )?, + + Operator::F64Copysign => { + let loc_b = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let loc_a = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let tmp1 = self.machine.acquire_temp_gpr().unwrap(); + let tmp2 = self.machine.acquire_temp_gpr().unwrap(); + let c = self.machine.acquire_temp_gpr().unwrap(); + + a.emit_mov(Size::S64, loc_a, Location::GPR(tmp1)); + a.emit_mov(Size::S64, loc_b, Location::GPR(tmp2)); + + a.emit_mov( + Size::S64, + Location::Imm64(0x7fffffffffffffffu64), + Location::GPR(c), + ); + a.emit_and(Size::S64, Location::GPR(c), Location::GPR(tmp1)); + + a.emit_mov( + Size::S64, + Location::Imm64(0x8000000000000000u64), + Location::GPR(c), + ); + a.emit_and(Size::S64, Location::GPR(c), Location::GPR(tmp2)); + + a.emit_or(Size::S64, Location::GPR(tmp2), Location::GPR(tmp1)); + a.emit_mov(Size::S64, Location::GPR(tmp1), ret); + + self.machine.release_temp_gpr(c); + self.machine.release_temp_gpr(tmp2); + self.machine.release_temp_gpr(tmp1); + } + + Operator::F64Abs => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let tmp = self.machine.acquire_temp_gpr().unwrap(); + let c = self.machine.acquire_temp_gpr().unwrap(); + + a.emit_mov(Size::S64, loc, Location::GPR(tmp)); + a.emit_mov( + Size::S64, + Location::Imm64(0x7fffffffffffffffu64), + Location::GPR(c), + ); + a.emit_and(Size::S64, Location::GPR(c), Location::GPR(tmp)); + a.emit_mov(Size::S64, Location::GPR(tmp), ret); + + self.machine.release_temp_gpr(c); + self.machine.release_temp_gpr(tmp); + } + + Operator::F64Neg => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + if a.arch_has_fneg() { + let tmp = self.machine.acquire_temp_xmm().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::XMM(tmp), + ); + a.arch_emit_f64_neg(tmp, tmp); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + Location::XMM(tmp), + ret, + ); + self.machine.release_temp_xmm(tmp); + } else { + let tmp = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S64, loc, Location::GPR(tmp)); + a.emit_btc_gpr_imm8_64(63, tmp); + a.emit_mov(Size::S64, Location::GPR(tmp), ret); + self.machine.release_temp_gpr(tmp); + } + } + + Operator::F64PromoteF32 => Self::emit_fp_unop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vcvtss2sd, + )?, + Operator::F32DemoteF64 => Self::emit_fp_unop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vcvtsd2ss, + )?, + + Operator::I32ReinterpretF32 => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + if loc != ret { + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + ret, + ); + } + } + Operator::F32ReinterpretI32 => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + if loc != ret { + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + ret, + ); + } + } + + Operator::I64ReinterpretF64 => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + if loc != ret { + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + ret, + ); + } + } + Operator::F64ReinterpretI64 => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + if loc != ret { + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + ret, + ); + } + } + + Operator::I32TruncF32U => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + if a.arch_has_itruncf() { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::XMM(tmp_in), + ); + a.arch_emit_i32_trunc_uf32(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + Location::GPR(tmp_out), + ret, + ); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::XMM(tmp_in), + ); + Self::emit_f32_int_conv_check_trap( + a, + &mut self.machine, + &mut self.exception_table, + tmp_in, + GEF32_LT_U32_MIN, + LEF32_GT_U32_MAX, + ); + + a.emit_cvttss2si_64(XMMOrMemory::XMM(tmp_in), tmp_out); + a.emit_mov(Size::S32, Location::GPR(tmp_out), ret); + + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + } + + Operator::I32TruncSatF32U => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::XMM(tmp_in), + ); + Self::emit_f32_int_conv_check_sat( + a, + &mut self.machine, + tmp_in, + GEF32_LT_U32_MIN, + LEF32_GT_U32_MAX, + |a, _m| { + a.emit_mov(Size::S32, Location::Imm32(0), Location::GPR(tmp_out)); + }, + |a, _m| { + a.emit_mov( + Size::S32, + Location::Imm32(std::u32::MAX), + Location::GPR(tmp_out), + ); + }, + None::, + |a, _m| { + if a.arch_has_itruncf() { + a.arch_emit_i32_trunc_uf32(tmp_in, tmp_out); + } else { + a.emit_cvttss2si_64(XMMOrMemory::XMM(tmp_in), tmp_out); + } + }, + ); + + a.emit_mov(Size::S32, Location::GPR(tmp_out), ret); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + + Operator::I32TruncF32S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + if a.arch_has_itruncf() { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::XMM(tmp_in), + ); + a.arch_emit_i32_trunc_sf32(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + Location::GPR(tmp_out), + ret, + ); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::XMM(tmp_in), + ); + Self::emit_f32_int_conv_check_trap( + a, + &mut self.machine, + &mut self.exception_table, + tmp_in, + GEF32_LT_I32_MIN, + LEF32_GT_I32_MAX, + ); + + a.emit_cvttss2si_32(XMMOrMemory::XMM(tmp_in), tmp_out); + a.emit_mov(Size::S32, Location::GPR(tmp_out), ret); + + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + } + Operator::I32TruncSatF32S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::XMM(tmp_in), + ); + Self::emit_f32_int_conv_check_sat( + a, + &mut self.machine, + tmp_in, + GEF32_LT_I32_MIN, + LEF32_GT_I32_MAX, + |a, _m| { + a.emit_mov( + Size::S32, + Location::Imm32(std::i32::MIN as u32), + Location::GPR(tmp_out), + ); + }, + |a, _m| { + a.emit_mov( + Size::S32, + Location::Imm32(std::i32::MAX as u32), + Location::GPR(tmp_out), + ); + }, + Some(|a: &mut Assembler, _m: &mut Machine| { + a.emit_mov(Size::S32, Location::Imm32(0), Location::GPR(tmp_out)); + }), + |a, _m| { + if a.arch_has_itruncf() { + a.arch_emit_i32_trunc_sf32(tmp_in, tmp_out); + } else { + a.emit_cvttss2si_32(XMMOrMemory::XMM(tmp_in), tmp_out); + } + }, + ); + + a.emit_mov(Size::S32, Location::GPR(tmp_out), ret); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + + Operator::I64TruncF32S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + if a.arch_has_itruncf() { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::XMM(tmp_in), + ); + a.arch_emit_i64_trunc_sf32(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + Location::GPR(tmp_out), + ret, + ); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::XMM(tmp_in), + ); + Self::emit_f32_int_conv_check_trap( + a, + &mut self.machine, + &mut self.exception_table, + tmp_in, + GEF32_LT_I64_MIN, + LEF32_GT_I64_MAX, + ); + a.emit_cvttss2si_64(XMMOrMemory::XMM(tmp_in), tmp_out); + a.emit_mov(Size::S64, Location::GPR(tmp_out), ret); + + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + } + + Operator::I64TruncSatF32S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::XMM(tmp_in), + ); + Self::emit_f32_int_conv_check_sat( + a, + &mut self.machine, + tmp_in, + GEF32_LT_I64_MIN, + LEF32_GT_I64_MAX, + |a, _m| { + a.emit_mov( + Size::S64, + Location::Imm64(std::i64::MIN as u64), + Location::GPR(tmp_out), + ); + }, + |a, _m| { + a.emit_mov( + Size::S64, + Location::Imm64(std::i64::MAX as u64), + Location::GPR(tmp_out), + ); + }, + Some(|a: &mut Assembler, _m: &mut Machine| { + a.emit_mov(Size::S64, Location::Imm64(0), Location::GPR(tmp_out)); + }), + |a, _m| { + if a.arch_has_itruncf() { + a.arch_emit_i64_trunc_sf32(tmp_in, tmp_out); + } else { + a.emit_cvttss2si_64(XMMOrMemory::XMM(tmp_in), tmp_out); + } + }, + ); + + a.emit_mov(Size::S64, Location::GPR(tmp_out), ret); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + + Operator::I64TruncF32U => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + if a.arch_has_itruncf() { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::XMM(tmp_in), + ); + a.arch_emit_i64_trunc_uf32(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + Location::GPR(tmp_out), + ret, + ); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); // xmm2 + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::XMM(tmp_in), + ); + Self::emit_f32_int_conv_check_trap( + a, + &mut self.machine, + &mut self.exception_table, + tmp_in, + GEF32_LT_U64_MIN, + LEF32_GT_U64_MAX, + ); + + let tmp = self.machine.acquire_temp_gpr().unwrap(); // r15 + let tmp_x1 = self.machine.acquire_temp_xmm().unwrap(); // xmm1 + let tmp_x2 = self.machine.acquire_temp_xmm().unwrap(); // xmm3 + + a.emit_mov( + Size::S32, + Location::Imm32(1593835520u32), + Location::GPR(tmp), + ); //float 9.22337203E+18 + a.emit_mov(Size::S32, Location::GPR(tmp), Location::XMM(tmp_x1)); + a.emit_mov(Size::S32, Location::XMM(tmp_in), Location::XMM(tmp_x2)); + a.emit_vsubss(tmp_in, XMMOrMemory::XMM(tmp_x1), tmp_in); + a.emit_cvttss2si_64(XMMOrMemory::XMM(tmp_in), tmp_out); + a.emit_mov( + Size::S64, + Location::Imm64(0x8000000000000000u64), + Location::GPR(tmp), + ); + a.emit_xor(Size::S64, Location::GPR(tmp_out), Location::GPR(tmp)); + a.emit_cvttss2si_64(XMMOrMemory::XMM(tmp_x2), tmp_out); + a.emit_ucomiss(XMMOrMemory::XMM(tmp_x1), tmp_x2); + a.emit_cmovae_gpr_64(tmp, tmp_out); + a.emit_mov(Size::S64, Location::GPR(tmp_out), ret); + + self.machine.release_temp_xmm(tmp_x2); + self.machine.release_temp_xmm(tmp_x1); + self.machine.release_temp_gpr(tmp); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + } + Operator::I64TruncSatF32U => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::XMM(tmp_in), + ); + Self::emit_f32_int_conv_check_sat( + a, + &mut self.machine, + tmp_in, + GEF32_LT_U64_MIN, + LEF32_GT_U64_MAX, + |a, _m| { + a.emit_mov(Size::S64, Location::Imm64(0), Location::GPR(tmp_out)); + }, + |a, _m| { + a.emit_mov( + Size::S64, + Location::Imm64(std::u64::MAX), + Location::GPR(tmp_out), + ); + }, + None::, + |a, m| { + if a.arch_has_itruncf() { + a.arch_emit_i64_trunc_uf32(tmp_in, tmp_out); + } else { + let tmp = m.acquire_temp_gpr().unwrap(); + let tmp_x1 = m.acquire_temp_xmm().unwrap(); + let tmp_x2 = m.acquire_temp_xmm().unwrap(); + + a.emit_mov( + Size::S32, + Location::Imm32(1593835520u32), + Location::GPR(tmp), + ); //float 9.22337203E+18 + a.emit_mov(Size::S32, Location::GPR(tmp), Location::XMM(tmp_x1)); + a.emit_mov(Size::S32, Location::XMM(tmp_in), Location::XMM(tmp_x2)); + a.emit_vsubss(tmp_in, XMMOrMemory::XMM(tmp_x1), tmp_in); + a.emit_cvttss2si_64(XMMOrMemory::XMM(tmp_in), tmp_out); + a.emit_mov( + Size::S64, + Location::Imm64(0x8000000000000000u64), + Location::GPR(tmp), + ); + a.emit_xor(Size::S64, Location::GPR(tmp_out), Location::GPR(tmp)); + a.emit_cvttss2si_64(XMMOrMemory::XMM(tmp_x2), tmp_out); + a.emit_ucomiss(XMMOrMemory::XMM(tmp_x1), tmp_x2); + a.emit_cmovae_gpr_64(tmp, tmp_out); + + m.release_temp_xmm(tmp_x2); + m.release_temp_xmm(tmp_x1); + m.release_temp_gpr(tmp); + } + }, + ); + + a.emit_mov(Size::S64, Location::GPR(tmp_out), ret); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + + Operator::I32TruncF64U => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + if a.arch_has_itruncf() { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::XMM(tmp_in), + ); + a.arch_emit_i32_trunc_uf64(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + Location::GPR(tmp_out), + ret, + ); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::XMM(tmp_in), + ); + Self::emit_f64_int_conv_check_trap( + a, + &mut self.machine, + &mut self.exception_table, + tmp_in, + GEF64_LT_U32_MIN, + LEF64_GT_U32_MAX, + ); + + a.emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_in), tmp_out); + a.emit_mov(Size::S32, Location::GPR(tmp_out), ret); + + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + } + + Operator::I32TruncSatF64U => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::XMM(tmp_in), + ); + Self::emit_f64_int_conv_check_sat( + a, + &mut self.machine, + tmp_in, + GEF64_LT_U32_MIN, + LEF64_GT_U32_MAX, + |a, _m| { + a.emit_mov(Size::S32, Location::Imm32(0), Location::GPR(tmp_out)); + }, + |a, _m| { + a.emit_mov( + Size::S32, + Location::Imm32(std::u32::MAX), + Location::GPR(tmp_out), + ); + }, + None::, + |a, _m| { + if a.arch_has_itruncf() { + a.arch_emit_i32_trunc_uf64(tmp_in, tmp_out); + } else { + a.emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_in), tmp_out); + } + }, + ); + + a.emit_mov(Size::S32, Location::GPR(tmp_out), ret); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + + Operator::I32TruncF64S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + if a.arch_has_itruncf() { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::XMM(tmp_in), + ); + a.arch_emit_i32_trunc_sf64(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + Location::GPR(tmp_out), + ret, + ); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + + let real_in = match loc { + Location::Imm32(_) | Location::Imm64(_) => { + a.emit_mov(Size::S64, loc, Location::GPR(tmp_out)); + a.emit_mov(Size::S64, Location::GPR(tmp_out), Location::XMM(tmp_in)); + tmp_in + } + Location::XMM(x) => x, + _ => { + a.emit_mov(Size::S64, loc, Location::XMM(tmp_in)); + tmp_in + } + }; + + Self::emit_f64_int_conv_check_trap( + a, + &mut self.machine, + &mut self.exception_table, + real_in, + GEF64_LT_I32_MIN, + LEF64_GT_I32_MAX, + ); + + a.emit_cvttsd2si_32(XMMOrMemory::XMM(real_in), tmp_out); + a.emit_mov(Size::S32, Location::GPR(tmp_out), ret); + + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + } + + Operator::I32TruncSatF64S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + + let real_in = match loc { + Location::Imm32(_) | Location::Imm64(_) => { + a.emit_mov(Size::S64, loc, Location::GPR(tmp_out)); + a.emit_mov(Size::S64, Location::GPR(tmp_out), Location::XMM(tmp_in)); + tmp_in + } + Location::XMM(x) => x, + _ => { + a.emit_mov(Size::S64, loc, Location::XMM(tmp_in)); + tmp_in + } + }; + + Self::emit_f64_int_conv_check_sat( + a, + &mut self.machine, + real_in, + GEF64_LT_I32_MIN, + LEF64_GT_I32_MAX, + |a, _m| { + a.emit_mov( + Size::S32, + Location::Imm32(std::i32::MIN as u32), + Location::GPR(tmp_out), + ); + }, + |a, _m| { + a.emit_mov( + Size::S32, + Location::Imm32(std::i32::MAX as u32), + Location::GPR(tmp_out), + ); + }, + Some(|a: &mut Assembler, _m: &mut Machine| { + a.emit_mov(Size::S32, Location::Imm32(0), Location::GPR(tmp_out)); + }), + |a, _m| { + if a.arch_has_itruncf() { + a.arch_emit_i32_trunc_sf64(tmp_in, tmp_out); + } else { + a.emit_cvttsd2si_32(XMMOrMemory::XMM(real_in), tmp_out); + } + }, + ); + + a.emit_mov(Size::S32, Location::GPR(tmp_out), ret); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + + Operator::I64TruncF64S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + if a.arch_has_itruncf() { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::XMM(tmp_in), + ); + a.arch_emit_i64_trunc_sf64(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + Location::GPR(tmp_out), + ret, + ); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::XMM(tmp_in), + ); + Self::emit_f64_int_conv_check_trap( + a, + &mut self.machine, + &mut self.exception_table, + tmp_in, + GEF64_LT_I64_MIN, + LEF64_GT_I64_MAX, + ); + + a.emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_in), tmp_out); + a.emit_mov(Size::S64, Location::GPR(tmp_out), ret); + + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + } + + Operator::I64TruncSatF64S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::XMM(tmp_in), + ); + Self::emit_f64_int_conv_check_sat( + a, + &mut self.machine, + tmp_in, + GEF64_LT_I64_MIN, + LEF64_GT_I64_MAX, + |a, _m| { + a.emit_mov( + Size::S64, + Location::Imm64(std::i64::MIN as u64), + Location::GPR(tmp_out), + ); + }, + |a, _m| { + a.emit_mov( + Size::S64, + Location::Imm64(std::i64::MAX as u64), + Location::GPR(tmp_out), + ); + }, + Some(|a: &mut Assembler, _m: &mut Machine| { + a.emit_mov(Size::S64, Location::Imm64(0), Location::GPR(tmp_out)); + }), + |a, _m| { + if a.arch_has_itruncf() { + a.arch_emit_i64_trunc_sf64(tmp_in, tmp_out); + } else { + a.emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_in), tmp_out); + } + }, + ); + + a.emit_mov(Size::S64, Location::GPR(tmp_out), ret); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + + Operator::I64TruncF64U => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + if a.arch_has_itruncf() { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::XMM(tmp_in), + ); + a.arch_emit_i64_trunc_uf64(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + Location::GPR(tmp_out), + ret, + ); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); // xmm2 + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::XMM(tmp_in), + ); + Self::emit_f64_int_conv_check_trap( + a, + &mut self.machine, + &mut self.exception_table, + tmp_in, + GEF64_LT_U64_MIN, + LEF64_GT_U64_MAX, + ); + + let tmp = self.machine.acquire_temp_gpr().unwrap(); // r15 + let tmp_x1 = self.machine.acquire_temp_xmm().unwrap(); // xmm1 + let tmp_x2 = self.machine.acquire_temp_xmm().unwrap(); // xmm3 + + a.emit_mov( + Size::S64, + Location::Imm64(4890909195324358656u64), + Location::GPR(tmp), + ); //double 9.2233720368547758E+18 + a.emit_mov(Size::S64, Location::GPR(tmp), Location::XMM(tmp_x1)); + a.emit_mov(Size::S64, Location::XMM(tmp_in), Location::XMM(tmp_x2)); + a.emit_vsubsd(tmp_in, XMMOrMemory::XMM(tmp_x1), tmp_in); + a.emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_in), tmp_out); + a.emit_mov( + Size::S64, + Location::Imm64(0x8000000000000000u64), + Location::GPR(tmp), + ); + a.emit_xor(Size::S64, Location::GPR(tmp_out), Location::GPR(tmp)); + a.emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_x2), tmp_out); + a.emit_ucomisd(XMMOrMemory::XMM(tmp_x1), tmp_x2); + a.emit_cmovae_gpr_64(tmp, tmp_out); + a.emit_mov(Size::S64, Location::GPR(tmp_out), ret); + + self.machine.release_temp_xmm(tmp_x2); + self.machine.release_temp_xmm(tmp_x1); + self.machine.release_temp_gpr(tmp); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + } + + Operator::I64TruncSatF64U => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::XMM(tmp_in), + ); + Self::emit_f64_int_conv_check_sat( + a, + &mut self.machine, + tmp_in, + GEF64_LT_U64_MIN, + LEF64_GT_U64_MAX, + |a, _m| { + a.emit_mov(Size::S64, Location::Imm64(0), Location::GPR(tmp_out)); + }, + |a, _m| { + a.emit_mov( + Size::S64, + Location::Imm64(std::u64::MAX), + Location::GPR(tmp_out), + ); + }, + None::, + |a, m| { + if a.arch_has_itruncf() { + a.arch_emit_i64_trunc_uf64(tmp_in, tmp_out); + } else { + let tmp = m.acquire_temp_gpr().unwrap(); + let tmp_x1 = m.acquire_temp_xmm().unwrap(); + let tmp_x2 = m.acquire_temp_xmm().unwrap(); + + a.emit_mov( + Size::S64, + Location::Imm64(4890909195324358656u64), + Location::GPR(tmp), + ); //double 9.2233720368547758E+18 + a.emit_mov(Size::S64, Location::GPR(tmp), Location::XMM(tmp_x1)); + a.emit_mov(Size::S64, Location::XMM(tmp_in), Location::XMM(tmp_x2)); + a.emit_vsubsd(tmp_in, XMMOrMemory::XMM(tmp_x1), tmp_in); + a.emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_in), tmp_out); + a.emit_mov( + Size::S64, + Location::Imm64(0x8000000000000000u64), + Location::GPR(tmp), + ); + a.emit_xor(Size::S64, Location::GPR(tmp_out), Location::GPR(tmp)); + a.emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_x2), tmp_out); + a.emit_ucomisd(XMMOrMemory::XMM(tmp_x1), tmp_x2); + a.emit_cmovae_gpr_64(tmp, tmp_out); + + m.release_temp_xmm(tmp_x2); + m.release_temp_xmm(tmp_x1); + m.release_temp_gpr(tmp); + } + }, + ); + + a.emit_mov(Size::S64, Location::GPR(tmp_out), ret); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + + Operator::F32ConvertI32S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + if a.arch_has_fconverti() { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::GPR(tmp_in), + ); + a.arch_emit_f32_convert_si32(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + Location::XMM(tmp_out), + ret, + ); + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + + a.emit_mov(Size::S32, loc, Location::GPR(tmp_in)); + a.emit_vcvtsi2ss_32(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out); + a.emit_mov(Size::S32, Location::XMM(tmp_out), ret); + + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } + } + Operator::F32ConvertI32U => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + if a.arch_has_fconverti() { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::GPR(tmp_in), + ); + a.arch_emit_f32_convert_ui32(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + Location::XMM(tmp_out), + ret, + ); + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + + a.emit_mov(Size::S32, loc, Location::GPR(tmp_in)); + a.emit_vcvtsi2ss_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out); + a.emit_mov(Size::S32, Location::XMM(tmp_out), ret); + + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } + } + Operator::F32ConvertI64S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + if a.arch_has_fconverti() { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::GPR(tmp_in), + ); + a.arch_emit_f32_convert_si64(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + Location::XMM(tmp_out), + ret, + ); + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + + a.emit_mov(Size::S64, loc, Location::GPR(tmp_in)); + a.emit_vcvtsi2ss_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out); + a.emit_mov(Size::S32, Location::XMM(tmp_out), ret); + + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } + } + Operator::F32ConvertI64U => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + if a.arch_has_fconverti() { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::GPR(tmp_in), + ); + a.arch_emit_f32_convert_ui64(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + Location::XMM(tmp_out), + ret, + ); + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + let tmp = self.machine.acquire_temp_gpr().unwrap(); + + let do_convert = a.get_label(); + let end_convert = a.get_label(); + + a.emit_mov(Size::S64, loc, Location::GPR(tmp_in)); + a.emit_test_gpr_64(tmp_in); + a.emit_jmp(Condition::Signed, do_convert); + a.emit_vcvtsi2ss_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out); + a.emit_jmp(Condition::None, end_convert); + a.emit_label(do_convert); + a.emit_mov(Size::S64, Location::GPR(tmp_in), Location::GPR(tmp)); + a.emit_and(Size::S64, Location::Imm32(1), Location::GPR(tmp)); + a.emit_shr(Size::S64, Location::Imm8(1), Location::GPR(tmp_in)); + a.emit_or(Size::S64, Location::GPR(tmp), Location::GPR(tmp_in)); + a.emit_vcvtsi2ss_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out); + a.emit_vaddss(tmp_out, XMMOrMemory::XMM(tmp_out), tmp_out); + a.emit_label(end_convert); + a.emit_mov(Size::S32, Location::XMM(tmp_out), ret); + + self.machine.release_temp_gpr(tmp); + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } + } + + Operator::F64ConvertI32S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + if a.arch_has_fconverti() { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::GPR(tmp_in), + ); + a.arch_emit_f64_convert_si32(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + Location::XMM(tmp_out), + ret, + ); + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + + a.emit_mov(Size::S32, loc, Location::GPR(tmp_in)); + a.emit_vcvtsi2sd_32(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out); + a.emit_mov(Size::S64, Location::XMM(tmp_out), ret); + + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } + } + Operator::F64ConvertI32U => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + if a.arch_has_fconverti() { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::GPR(tmp_in), + ); + a.arch_emit_f64_convert_ui32(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + Location::XMM(tmp_out), + ret, + ); + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + + a.emit_mov(Size::S32, loc, Location::GPR(tmp_in)); + a.emit_vcvtsi2sd_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out); + a.emit_mov(Size::S64, Location::XMM(tmp_out), ret); + + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } + } + Operator::F64ConvertI64S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + if a.arch_has_fconverti() { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::GPR(tmp_in), + ); + a.arch_emit_f64_convert_si64(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + Location::XMM(tmp_out), + ret, + ); + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + + a.emit_mov(Size::S64, loc, Location::GPR(tmp_in)); + a.emit_vcvtsi2sd_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out); + a.emit_mov(Size::S64, Location::XMM(tmp_out), ret); + + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } + } + Operator::F64ConvertI64U => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + if a.arch_has_fconverti() { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::GPR(tmp_in), + ); + a.arch_emit_f64_convert_ui64(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + Location::XMM(tmp_out), + ret, + ); + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + let tmp = self.machine.acquire_temp_gpr().unwrap(); + + let do_convert = a.get_label(); + let end_convert = a.get_label(); + + a.emit_mov(Size::S64, loc, Location::GPR(tmp_in)); + a.emit_test_gpr_64(tmp_in); + a.emit_jmp(Condition::Signed, do_convert); + a.emit_vcvtsi2sd_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out); + a.emit_jmp(Condition::None, end_convert); + a.emit_label(do_convert); + a.emit_mov(Size::S64, Location::GPR(tmp_in), Location::GPR(tmp)); + a.emit_and(Size::S64, Location::Imm32(1), Location::GPR(tmp)); + a.emit_shr(Size::S64, Location::Imm8(1), Location::GPR(tmp_in)); + a.emit_or(Size::S64, Location::GPR(tmp), Location::GPR(tmp_in)); + a.emit_vcvtsi2sd_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out); + a.emit_vaddsd(tmp_out, XMMOrMemory::XMM(tmp_out), tmp_out); + a.emit_label(end_convert); + a.emit_mov(Size::S64, Location::XMM(tmp_out), ret); + + self.machine.release_temp_gpr(tmp); + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } + } + + Operator::Call { function_index } => { + let function_index = function_index as usize; + let label = self + .function_labels + .as_mut() + .unwrap() + .entry(function_index) + .or_insert_with(|| (a.get_label(), None)) + .0; + let sig_index = *self + .function_signatures + .get(FuncIndex::new(function_index)) + .unwrap(); + let sig = self.signatures.get(sig_index).unwrap(); + let param_types: SmallVec<[WpType; 8]> = + sig.params().iter().cloned().map(type_to_wp_type).collect(); + let return_types: SmallVec<[WpType; 1]> = + sig.returns().iter().cloned().map(type_to_wp_type).collect(); + + let params: SmallVec<[_; 8]> = self + .value_stack + .drain(self.value_stack.len() - param_types.len()..) + .collect(); + self.machine.release_locations_only_regs(¶ms); + + self.machine.release_locations_only_osr_state(params.len()); + + Self::emit_call_sysv_label( + a, + &mut self.machine, + label, + params.iter().map(|x| *x), + Some((&mut self.fsm, &mut self.control_stack)), + )?; + + self.machine.release_locations_only_stack(a, ¶ms); + + if return_types.len() > 0 { + let ret = self.machine.acquire_locations( + a, + &[( + return_types[0], + MachineValue::WasmStack(self.value_stack.len()), + )], + false, + )[0]; + self.value_stack.push(ret); + a.emit_mov(Size::S64, Location::GPR(GPR::RAX), ret); + } + } + Operator::CallIndirect { index, table_index } => { + if table_index != 0 { + return Err(CodegenError { + message: format!("CallIndirect: table_index is not 0"), + }); + } + let sig = self.signatures.get(SigIndex::new(index as usize)).unwrap(); + let param_types: SmallVec<[WpType; 8]> = + sig.params().iter().cloned().map(type_to_wp_type).collect(); + let return_types: SmallVec<[WpType; 1]> = + sig.returns().iter().cloned().map(type_to_wp_type).collect(); + + let func_index = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + let params: SmallVec<[_; 8]> = self + .value_stack + .drain(self.value_stack.len() - param_types.len()..) + .collect(); + self.machine.release_locations_only_regs(¶ms); + + let table_base = self.machine.acquire_temp_gpr().unwrap(); + let table_count = self.machine.acquire_temp_gpr().unwrap(); + let sigidx = self.machine.acquire_temp_gpr().unwrap(); + + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + match TableIndex::new(0).local_or_import(module_info) { + LocalOrImport::Local(_) => vm::Ctx::offset_tables(), + LocalOrImport::Import(_) => vm::Ctx::offset_imported_tables(), + } as i32, + ), + Location::GPR(table_base), + ); + a.emit_mov( + Size::S64, + Location::Memory(table_base, 0), + Location::GPR(table_base), + ); + a.emit_mov( + Size::S32, + Location::Memory(table_base, LocalTable::offset_count() as i32), + Location::GPR(table_count), + ); + a.emit_mov( + Size::S64, + Location::Memory(table_base, LocalTable::offset_base() as i32), + Location::GPR(table_base), + ); + a.emit_cmp(Size::S32, func_index, Location::GPR(table_count)); + + match &mut self.exception_table { + Some(etable) => { + Self::mark_range_with_exception_code( + a, + etable, + ExceptionCode::CallIndirectOOB, + |a| a.emit_conditional_trap(Condition::BelowEqual), + ); + } + None => { a.emit_conditional_trap(Condition::BelowEqual); } + }; + + a.emit_mov(Size::S32, func_index, Location::GPR(table_count)); + a.emit_imul_imm32_gpr64(vm::Anyfunc::size() as u32, table_count); + a.emit_add( + Size::S64, + Location::GPR(table_base), + Location::GPR(table_count), + ); + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + vm::Ctx::offset_signatures() as i32, + ), + Location::GPR(sigidx), + ); + a.emit_mov( + Size::S32, + Location::Memory(sigidx, (index * 4) as i32), + Location::GPR(sigidx), + ); + a.emit_cmp( + Size::S32, + Location::GPR(sigidx), + Location::Memory(table_count, (vm::Anyfunc::offset_sig_id() as usize) as i32), + ); + match &mut self.exception_table { + Some(etable) => { + Self::mark_range_with_exception_code( + a, + etable, + ExceptionCode::IncorrectCallIndirectSignature, + |a| a.emit_conditional_trap(Condition::NotEqual), + ); + } + None => { a.emit_conditional_trap(Condition::NotEqual); } + } + + self.machine.release_temp_gpr(sigidx); + self.machine.release_temp_gpr(table_count); + self.machine.release_temp_gpr(table_base); + + if table_count != GPR::RAX { + a.emit_mov( + Size::S64, + Location::GPR(table_count), + Location::GPR(GPR::RAX), + ); + } + + self.machine.release_locations_only_osr_state(params.len()); + + Self::emit_call_sysv( + a, + &mut self.machine, + |a| { + if a.arch_requires_indirect_call_trampoline() { + a.arch_emit_indirect_call_with_trampoline(Location::Memory( + GPR::RAX, + (vm::Anyfunc::offset_func() as usize) as i32, + )); + } else { + a.emit_call_location(Location::Memory( + GPR::RAX, + (vm::Anyfunc::offset_func() as usize) as i32, + )); + } + }, + params.iter().map(|x| *x), + Some((&mut self.fsm, &mut self.control_stack)), + )?; + + self.machine.release_locations_only_stack(a, ¶ms); + + if return_types.len() > 0 { + let ret = self.machine.acquire_locations( + a, + &[( + return_types[0], + MachineValue::WasmStack(self.value_stack.len()), + )], + false, + )[0]; + self.value_stack.push(ret); + a.emit_mov(Size::S64, Location::GPR(GPR::RAX), ret); + } + } + Operator::If { ty } => { + let label_end = a.get_label(); + let label_else = a.get_label(); + + let cond = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + let frame = ControlFrame { + label: label_end, + loop_like: false, + if_else: IfElseState::If(label_else), + returns: match ty { + WpTypeOrFuncType::Type(WpType::EmptyBlockType) => smallvec![], + WpTypeOrFuncType::Type(inner_ty) => smallvec![inner_ty], + _ => { + return Err(CodegenError { + message: format!("If: multi-value returns not yet implemented"), + }) + } + }, + value_stack_depth: self.value_stack.len(), + state: self.machine.state.clone(), + state_diff_id: Self::get_state_diff( + &self.machine, + &mut self.fsm, + &mut self.control_stack, + ), + }; + self.control_stack.push(frame); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_cmp, + Size::S32, + Location::Imm32(0), + cond, + ); + a.emit_jmp(Condition::Equal, label_else); + } + Operator::Else => { + let mut frame = self.control_stack.last_mut().unwrap(); + + if !was_unreachable && frame.returns.len() > 0 { + let loc = *self.value_stack.last().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::GPR(GPR::RAX), + ); + } + + let released: &[Location] = &self.value_stack[frame.value_stack_depth..]; + self.machine.release_locations(a, released); + self.value_stack.truncate(frame.value_stack_depth); + + match frame.if_else { + IfElseState::If(label) => { + a.emit_jmp(Condition::None, frame.label); + a.emit_label(label); + frame.if_else = IfElseState::Else; + } + _ => { + return Err(CodegenError { + message: format!("Else: frame.if_else unreachable code"), + }) + } + } + } + Operator::Select => { + let cond = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let v_b = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let v_a = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let end_label = a.get_label(); + let zero_label = a.get_label(); + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_cmp, + Size::S32, + Location::Imm32(0), + cond, + ); + a.emit_jmp(Condition::Equal, zero_label); + if v_a != ret { + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + v_a, + ret, + ); + } + a.emit_jmp(Condition::None, end_label); + a.emit_label(zero_label); + if v_b != ret { + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + v_b, + ret, + ); + } + a.emit_label(end_label); + } + Operator::Block { ty } => { + let frame = ControlFrame { + label: a.get_label(), + loop_like: false, + if_else: IfElseState::None, + returns: match ty { + WpTypeOrFuncType::Type(WpType::EmptyBlockType) => smallvec![], + WpTypeOrFuncType::Type(inner_ty) => smallvec![inner_ty], + _ => { + return Err(CodegenError { + message: format!("Block: multi-value returns not yet implemented"), + }) + } + }, + value_stack_depth: self.value_stack.len(), + state: self.machine.state.clone(), + state_diff_id: Self::get_state_diff( + &self.machine, + &mut self.fsm, + &mut self.control_stack, + ), + }; + self.control_stack.push(frame); + } + Operator::Loop { ty } => { + let label = a.get_label(); + let state_diff_id = + Self::get_state_diff(&self.machine, &mut self.fsm, &mut self.control_stack); + let activate_offset = a.get_offset().0; + + self.control_stack.push(ControlFrame { + label: label, + loop_like: true, + if_else: IfElseState::None, + returns: match ty { + WpTypeOrFuncType::Type(WpType::EmptyBlockType) => smallvec![], + WpTypeOrFuncType::Type(inner_ty) => smallvec![inner_ty], + _ => { + return Err(CodegenError { + message: format!("Loop: multi-value returns not yet implemented"), + }) + } + }, + value_stack_depth: self.value_stack.len(), + state: self.machine.state.clone(), + state_diff_id, + }); + a.emit_label(label); + + // Check interrupt signal without branching + if self.config.full_preemption { + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + vm::Ctx::offset_interrupt_signal_mem() as i32, + ), + Location::GPR(GPR::RAX), + ); + self.fsm.loop_offsets.insert( + a.get_offset().0, + OffsetInfo { + end_offset: a.get_offset().0 + 1, + activate_offset, + diff_id: state_diff_id, + }, + ); + self.fsm.wasm_offset_to_target_offset.insert( + self.machine.state.wasm_inst_offset, + SuspendOffset::Loop(a.get_offset().0), + ); + a.emit_mov( + Size::S64, + Location::Memory(GPR::RAX, 0), + Location::GPR(GPR::RAX), + ); + } + } + Operator::Nop => {} + Operator::MemorySize { reserved } => { + let memory_index = MemoryIndex::new(reserved as usize); + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + vm::Ctx::offset_intrinsics() as i32, + ), + Location::GPR(GPR::RAX), + ); + a.emit_mov( + Size::S64, + Location::Memory(GPR::RAX, vm::Intrinsics::offset_memory_size() as i32), + Location::GPR(GPR::RAX), + ); + Self::emit_call_sysv( + a, + &mut self.machine, + |a| { + let label = a.get_label(); + let after = a.get_label(); + a.emit_jmp(Condition::None, after); + a.emit_label(label); + a.emit_host_redirection(GPR::RAX); + a.emit_label(after); + a.emit_call_label(label); + }, + iter::once(Location::Imm32(memory_index.index() as u32)), + None, + )?; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + a.emit_mov(Size::S64, Location::GPR(GPR::RAX), ret); + } + Operator::MemoryGrow { reserved } => { + let memory_index = MemoryIndex::new(reserved as usize); + let param_pages = self.value_stack.pop().unwrap(); + + self.machine.release_locations_only_regs(&[param_pages]); + + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + vm::Ctx::offset_intrinsics() as i32, + ), + Location::GPR(GPR::RAX), + ); + a.emit_mov( + Size::S64, + Location::Memory(GPR::RAX, vm::Intrinsics::offset_memory_grow() as i32), + Location::GPR(GPR::RAX), + ); + + self.machine.release_locations_only_osr_state(1); + + Self::emit_call_sysv( + a, + &mut self.machine, + |a| { + let label = a.get_label(); + let after = a.get_label(); + a.emit_jmp(Condition::None, after); + a.emit_label(label); + a.emit_host_redirection(GPR::RAX); + a.emit_label(after); + a.emit_call_label(label); + }, + iter::once(Location::Imm32(memory_index.index() as u32)) + .chain(iter::once(param_pages)), + None, + )?; + + self.machine.release_locations_only_stack(a, &[param_pages]); + + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + a.emit_mov(Size::S64, Location::GPR(GPR::RAX), ret); + } + Operator::I32Load { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + false, + 4, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S32, + Location::Memory(addr, 0), + ret, + ); + Ok(()) + }, + )?; + } + Operator::F32Load { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + false, + 4, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S32, + Location::Memory(addr, 0), + ret, + ); + Ok(()) + }, + )?; + } + Operator::I32Load8U { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + false, + 1, + |a, m, addr| { + Self::emit_relaxed_zx_sx( + a, + m, + Assembler::emit_movzx, + Size::S8, + Location::Memory(addr, 0), + Size::S32, + ret, + )?; + Ok(()) + }, + )?; + } + Operator::I32Load8S { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + false, + 1, + |a, m, addr| { + Self::emit_relaxed_zx_sx( + a, + m, + Assembler::emit_movsx, + Size::S8, + Location::Memory(addr, 0), + Size::S32, + ret, + )?; + Ok(()) + }, + )?; + } + Operator::I32Load16U { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + false, + 2, + |a, m, addr| { + Self::emit_relaxed_zx_sx( + a, + m, + Assembler::emit_movzx, + Size::S16, + Location::Memory(addr, 0), + Size::S32, + ret, + )?; + Ok(()) + }, + )?; + } + Operator::I32Load16S { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + false, + 2, + |a, m, addr| { + Self::emit_relaxed_zx_sx( + a, + m, + Assembler::emit_movsx, + Size::S16, + Location::Memory(addr, 0), + Size::S32, + ret, + )?; + Ok(()) + }, + )?; + } + Operator::I32Store { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target_addr, + memarg, + false, + 4, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S32, + target_value, + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + } + Operator::F32Store { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target_addr, + memarg, + false, + 4, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S32, + target_value, + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + } + Operator::I32Store8 { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target_addr, + memarg, + false, + 1, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S8, + target_value, + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + } + Operator::I32Store16 { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target_addr, + memarg, + false, + 2, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S16, + target_value, + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + } + Operator::I64Load { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + false, + 8, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S64, + Location::Memory(addr, 0), + ret, + ); + Ok(()) + }, + )?; + } + Operator::F64Load { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + false, + 8, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S64, + Location::Memory(addr, 0), + ret, + ); + Ok(()) + }, + )?; + } + Operator::I64Load8U { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + false, + 1, + |a, m, addr| { + Self::emit_relaxed_zx_sx( + a, + m, + Assembler::emit_movzx, + Size::S8, + Location::Memory(addr, 0), + Size::S64, + ret, + )?; + Ok(()) + }, + )?; + } + Operator::I64Load8S { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + false, + 1, + |a, m, addr| { + Self::emit_relaxed_zx_sx( + a, + m, + Assembler::emit_movsx, + Size::S8, + Location::Memory(addr, 0), + Size::S64, + ret, + )?; + Ok(()) + }, + )?; + } + Operator::I64Load16U { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + false, + 2, + |a, m, addr| { + Self::emit_relaxed_zx_sx( + a, + m, + Assembler::emit_movzx, + Size::S16, + Location::Memory(addr, 0), + Size::S64, + ret, + )?; + Ok(()) + }, + )?; + } + Operator::I64Load16S { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + false, + 2, + |a, m, addr| { + Self::emit_relaxed_zx_sx( + a, + m, + Assembler::emit_movsx, + Size::S16, + Location::Memory(addr, 0), + Size::S64, + ret, + )?; + Ok(()) + }, + )?; + } + Operator::I64Load32U { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + false, + 4, + |a, m, addr| { + match ret { + Location::GPR(_) => {} + Location::Memory(base, offset) => { + a.emit_mov( + Size::S32, + Location::Imm32(0), + Location::Memory(base, offset + 4), + ); // clear upper bits + } + _ => { + return Err(CodegenError { + message: format!("I64Load32U ret: unreachable code"), + }) + } + } + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S32, + Location::Memory(addr, 0), + ret, + ); + Ok(()) + }, + )?; + } + Operator::I64Load32S { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + false, + 4, + |a, m, addr| { + Self::emit_relaxed_zx_sx( + a, + m, + Assembler::emit_movsx, + Size::S32, + Location::Memory(addr, 0), + Size::S64, + ret, + )?; + Ok(()) + }, + )?; + } + Operator::I64Store { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target_addr, + memarg, + false, + 8, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S64, + target_value, + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + } + Operator::F64Store { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target_addr, + memarg, + false, + 8, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S64, + target_value, + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + } + Operator::I64Store8 { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target_addr, + memarg, + false, + 1, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S8, + target_value, + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + } + Operator::I64Store16 { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target_addr, + memarg, + false, + 2, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S16, + target_value, + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + } + Operator::I64Store32 { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target_addr, + memarg, + false, + 4, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S32, + target_value, + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + } + Operator::Unreachable => { + Self::mark_trappable(a, &self.machine, &mut self.fsm, &mut self.control_stack); + match &mut self.exception_table { + Some(etable) => { + etable.offset_to_code.insert(a.get_offset().0, ExceptionCode::Unreachable); + } + None => {} + }; + a.emit_ud2(); + self.unreachable_depth = 1; + } + Operator::Return => { + let frame = &self.control_stack[0]; + if frame.returns.len() > 0 { + if frame.returns.len() != 1 { + return Err(CodegenError { + message: format!("Return: incorrect frame.returns"), + }); + } + let loc = *self.value_stack.last().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::GPR(GPR::RAX), + ); + } + let released = &self.value_stack[frame.value_stack_depth..]; + self.machine.release_locations_keep_state(a, released); + a.emit_jmp(Condition::None, frame.label); + self.unreachable_depth = 1; + } + Operator::Br { relative_depth } => { + let frame = + &self.control_stack[self.control_stack.len() - 1 - (relative_depth as usize)]; + if !frame.loop_like && frame.returns.len() > 0 { + if frame.returns.len() != 1 { + return Err(CodegenError { + message: format!("Br: incorrect frame.returns"), + }); + } + let loc = *self.value_stack.last().unwrap(); + a.emit_mov(Size::S64, loc, Location::GPR(GPR::RAX)); + } + let released = &self.value_stack[frame.value_stack_depth..]; + self.machine.release_locations_keep_state(a, released); + a.emit_jmp(Condition::None, frame.label); + self.unreachable_depth = 1; + } + Operator::BrIf { relative_depth } => { + let after = a.get_label(); + let cond = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_cmp, + Size::S32, + Location::Imm32(0), + cond, + ); + a.emit_jmp(Condition::Equal, after); + + let frame = + &self.control_stack[self.control_stack.len() - 1 - (relative_depth as usize)]; + if !frame.loop_like && frame.returns.len() > 0 { + if frame.returns.len() != 1 { + return Err(CodegenError { + message: format!("BrIf: incorrect frame.returns"), + }); + } + let loc = *self.value_stack.last().unwrap(); + a.emit_mov(Size::S64, loc, Location::GPR(GPR::RAX)); + } + let released = &self.value_stack[frame.value_stack_depth..]; + self.machine.release_locations_keep_state(a, released); + a.emit_jmp(Condition::None, frame.label); + + a.emit_label(after); + } + Operator::BrTable { ref table } => { + let (targets, default_target) = table.read_table().map_err(|e| CodegenError { + message: format!("BrTable read_table: {:?}", e), + })?; + let cond = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let table_label = a.get_label(); + let mut table: Vec = vec![]; + let default_br = a.get_label(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_cmp, + Size::S32, + Location::Imm32(targets.len() as u32), + cond, + ); + a.emit_jmp(Condition::AboveEqual, default_br); + + a.emit_lea_label(table_label, Location::GPR(GPR::RCX)); + a.emit_mov(Size::S32, cond, Location::GPR(GPR::RDX)); + + let instr_size = a.get_jmp_instr_size(); + a.emit_imul_imm32_gpr64(instr_size as _, GPR::RDX); + a.emit_add(Size::S64, Location::GPR(GPR::RCX), Location::GPR(GPR::RDX)); + a.emit_jmp_location(Location::GPR(GPR::RDX)); + + for target in targets.iter() { + let label = a.get_label(); + a.emit_label(label); + table.push(label); + let frame = + &self.control_stack[self.control_stack.len() - 1 - (*target as usize)]; + if !frame.loop_like && frame.returns.len() > 0 { + if frame.returns.len() != 1 { + return Err(CodegenError { + message: format!( + "BrTable: incorrect frame.returns for {:?}", + target + ), + }); + } + let loc = *self.value_stack.last().unwrap(); + a.emit_mov(Size::S64, loc, Location::GPR(GPR::RAX)); + } + let released = &self.value_stack[frame.value_stack_depth..]; + self.machine.release_locations_keep_state(a, released); + a.emit_jmp(Condition::None, frame.label); + } + a.emit_label(default_br); + + { + let frame = &self.control_stack + [self.control_stack.len() - 1 - (default_target as usize)]; + if !frame.loop_like && frame.returns.len() > 0 { + if frame.returns.len() != 1 { + return Err(CodegenError { + message: format!("BrTable: incorrect frame.returns"), + }); + } + let loc = *self.value_stack.last().unwrap(); + a.emit_mov(Size::S64, loc, Location::GPR(GPR::RAX)); + } + let released = &self.value_stack[frame.value_stack_depth..]; + self.machine.release_locations_keep_state(a, released); + a.emit_jmp(Condition::None, frame.label); + } + + a.emit_label(table_label); + for x in table { + a.emit_jmp(Condition::None, x); + } + self.unreachable_depth = 1; + } + Operator::Drop => { + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + } + Operator::End => { + let frame = self.control_stack.pop().unwrap(); + + if !was_unreachable && frame.returns.len() > 0 { + let loc = *self.value_stack.last().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::GPR(GPR::RAX), + ); + } + + if self.control_stack.len() == 0 { + a.emit_label(frame.label); + self.machine.finalize_locals(a, &self.locals); + a.emit_mov(Size::S64, Location::GPR(GPR::RBP), Location::GPR(GPR::RSP)); + a.emit_pop(Size::S64, Location::GPR(GPR::RBP)); + a.emit_ret(); + } else { + let released = &self.value_stack[frame.value_stack_depth..]; + self.machine.release_locations(a, released); + self.value_stack.truncate(frame.value_stack_depth); + + if !frame.loop_like { + a.emit_label(frame.label); + } + + if let IfElseState::If(label) = frame.if_else { + a.emit_label(label); + } + + if frame.returns.len() > 0 { + if frame.returns.len() != 1 { + return Err(CodegenError { + message: format!("End: incorrect frame.returns"), + }); + } + let loc = self.machine.acquire_locations( + a, + &[( + frame.returns[0], + MachineValue::WasmStack(self.value_stack.len()), + )], + false, + )[0]; + a.emit_mov(Size::S64, Location::GPR(GPR::RAX), loc); + self.value_stack.push(loc); + } + } + } + Operator::AtomicFence { flags: _ } => { + // Fence is a nop. + // + // Fence was added to preserve information about fences from + // source languages. If in the future Wasm extends the memory + // model, and if we hadn't recorded what fences used to be there, + // it would lead to data races that weren't present in the + // original source language. + } + Operator::I32AtomicLoad { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 4, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S32, + Location::Memory(addr, 0), + ret, + ); + Ok(()) + }, + )?; + } + Operator::I32AtomicLoad8U { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 1, + |a, m, addr| { + Self::emit_relaxed_zx_sx( + a, + m, + Assembler::emit_movzx, + Size::S8, + Location::Memory(addr, 0), + Size::S32, + ret, + )?; + Ok(()) + }, + )?; + } + Operator::I32AtomicLoad16U { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 2, + |a, m, addr| { + Self::emit_relaxed_zx_sx( + a, + m, + Assembler::emit_movzx, + Size::S16, + Location::Memory(addr, 0), + Size::S32, + ret, + )?; + Ok(()) + }, + )?; + } + Operator::I32AtomicStore { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target_addr, + memarg, + true, + 4, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_xchg, + Size::S32, + target_value, + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + } + Operator::I32AtomicStore8 { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target_addr, + memarg, + true, + 1, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_xchg, + Size::S8, + target_value, + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + } + Operator::I32AtomicStore16 { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target_addr, + memarg, + true, + 2, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_xchg, + Size::S16, + target_value, + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + } + Operator::I64AtomicLoad { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 8, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S64, + Location::Memory(addr, 0), + ret, + ); + Ok(()) + }, + )?; + } + Operator::I64AtomicLoad8U { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 1, + |a, m, addr| { + Self::emit_relaxed_zx_sx( + a, + m, + Assembler::emit_movzx, + Size::S8, + Location::Memory(addr, 0), + Size::S64, + ret, + )?; + Ok(()) + }, + )?; + } + Operator::I64AtomicLoad16U { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 2, + |a, m, addr| { + Self::emit_relaxed_zx_sx( + a, + m, + Assembler::emit_movzx, + Size::S16, + Location::Memory(addr, 0), + Size::S64, + ret, + )?; + Ok(()) + }, + )?; + } + Operator::I64AtomicLoad32U { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 4, + |a, m, addr| { + match ret { + Location::GPR(_) => {} + Location::Memory(base, offset) => { + a.emit_mov( + Size::S32, + Location::Imm32(0), + Location::Memory(base, offset + 4), + ); // clear upper bits + } + _ => { + return Err(CodegenError { + message: format!("I64AtomicLoad32U ret: unreachable code"), + }) + } + } + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S32, + Location::Memory(addr, 0), + ret, + ); + Ok(()) + }, + )?; + } + Operator::I64AtomicStore { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target_addr, + memarg, + true, + 8, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_xchg, + Size::S64, + target_value, + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + } + Operator::I64AtomicStore8 { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target_addr, + memarg, + true, + 1, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_xchg, + Size::S8, + target_value, + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + } + Operator::I64AtomicStore16 { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target_addr, + memarg, + true, + 2, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_xchg, + Size::S16, + target_value, + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + } + Operator::I64AtomicStore32 { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target_addr, + memarg, + true, + 4, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_xchg, + Size::S32, + target_value, + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + } + Operator::I32AtomicRmwAdd { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S32, loc, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 4, + |a, _m, addr| { + a.emit_lock_xadd( + Size::S32, + Location::GPR(value), + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + a.emit_mov(Size::S32, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmwAdd { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S64, loc, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 8, + |a, _m, addr| { + a.emit_lock_xadd( + Size::S64, + Location::GPR(value), + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I32AtomicRmw8AddU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S8, loc, Size::S32, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 1, + |a, _m, addr| { + a.emit_lock_xadd(Size::S8, Location::GPR(value), Location::Memory(addr, 0)); + Ok(()) + }, + )?; + a.emit_mov(Size::S32, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I32AtomicRmw16AddU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S16, loc, Size::S32, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 2, + |a, _m, addr| { + a.emit_lock_xadd( + Size::S16, + Location::GPR(value), + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + a.emit_mov(Size::S32, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmw8AddU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S8, loc, Size::S64, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 1, + |a, _m, addr| { + a.emit_lock_xadd(Size::S8, Location::GPR(value), Location::Memory(addr, 0)); + Ok(()) + }, + )?; + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmw16AddU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S16, loc, Size::S64, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 2, + |a, _m, addr| { + a.emit_lock_xadd( + Size::S16, + Location::GPR(value), + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmw32AddU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S32, loc, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 4, + |a, _m, addr| { + a.emit_lock_xadd( + Size::S32, + Location::GPR(value), + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I32AtomicRmwSub { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S32, loc, Location::GPR(value)); + a.emit_neg(Size::S32, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 4, + |a, _m, addr| { + a.emit_lock_xadd( + Size::S32, + Location::GPR(value), + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + a.emit_mov(Size::S32, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmwSub { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S64, loc, Location::GPR(value)); + a.emit_neg(Size::S64, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 8, + |a, _m, addr| { + a.emit_lock_xadd( + Size::S64, + Location::GPR(value), + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I32AtomicRmw8SubU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S8, loc, Size::S32, Location::GPR(value)); + a.emit_neg(Size::S8, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 1, + |a, _m, addr| { + a.emit_lock_xadd(Size::S8, Location::GPR(value), Location::Memory(addr, 0)); + Ok(()) + }, + )?; + a.emit_mov(Size::S32, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I32AtomicRmw16SubU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S16, loc, Size::S32, Location::GPR(value)); + a.emit_neg(Size::S16, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 2, + |a, _m, addr| { + a.emit_lock_xadd( + Size::S16, + Location::GPR(value), + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + a.emit_mov(Size::S32, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmw8SubU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S8, loc, Size::S64, Location::GPR(value)); + a.emit_neg(Size::S8, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 1, + |a, _m, addr| { + a.emit_lock_xadd(Size::S8, Location::GPR(value), Location::Memory(addr, 0)); + Ok(()) + }, + )?; + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmw16SubU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S16, loc, Size::S64, Location::GPR(value)); + a.emit_neg(Size::S16, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 2, + |a, _m, addr| { + a.emit_lock_xadd( + Size::S16, + Location::GPR(value), + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmw32SubU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S32, loc, Location::GPR(value)); + a.emit_neg(Size::S32, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 2, + |a, _m, addr| { + a.emit_lock_xadd( + Size::S32, + Location::GPR(value), + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I32AtomicRmwAnd { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + loc, + target, + ret, + memarg, + 4, + Size::S32, + Size::S32, + |a, _m, src, dst| { + a.emit_and(Size::S32, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I64AtomicRmwAnd { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + loc, + target, + ret, + memarg, + 8, + Size::S64, + Size::S64, + |a, _m, src, dst| { + a.emit_and(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I32AtomicRmw8AndU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + loc, + target, + ret, + memarg, + 1, + Size::S8, + Size::S32, + |a, _m, src, dst| { + a.emit_and(Size::S32, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I32AtomicRmw16AndU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + loc, + target, + ret, + memarg, + 1, + Size::S16, + Size::S32, + |a, _m, src, dst| { + a.emit_and(Size::S32, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I64AtomicRmw8AndU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + loc, + target, + ret, + memarg, + 1, + Size::S8, + Size::S64, + |a, _m, src, dst| { + a.emit_and(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I64AtomicRmw16AndU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + loc, + target, + ret, + memarg, + 1, + Size::S16, + Size::S64, + |a, _m, src, dst| { + a.emit_and(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I64AtomicRmw32AndU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + loc, + target, + ret, + memarg, + 1, + Size::S32, + Size::S64, + |a, _m, src, dst| { + a.emit_and(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I32AtomicRmwOr { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + loc, + target, + ret, + memarg, + 4, + Size::S32, + Size::S32, + |a, _m, src, dst| { + a.emit_or(Size::S32, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I64AtomicRmwOr { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + loc, + target, + ret, + memarg, + 8, + Size::S64, + Size::S64, + |a, _m, src, dst| { + a.emit_or(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I32AtomicRmw8OrU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + loc, + target, + ret, + memarg, + 1, + Size::S8, + Size::S32, + |a, _m, src, dst| { + a.emit_or(Size::S32, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I32AtomicRmw16OrU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + loc, + target, + ret, + memarg, + 1, + Size::S16, + Size::S32, + |a, _m, src, dst| { + a.emit_or(Size::S32, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I64AtomicRmw8OrU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + loc, + target, + ret, + memarg, + 1, + Size::S8, + Size::S64, + |a, _m, src, dst| { + a.emit_or(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I64AtomicRmw16OrU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + loc, + target, + ret, + memarg, + 1, + Size::S16, + Size::S64, + |a, _m, src, dst| { + a.emit_or(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I64AtomicRmw32OrU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + loc, + target, + ret, + memarg, + 1, + Size::S32, + Size::S64, + |a, _m, src, dst| { + a.emit_or(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I32AtomicRmwXor { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + loc, + target, + ret, + memarg, + 4, + Size::S32, + Size::S32, + |a, _m, src, dst| { + a.emit_xor(Size::S32, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I64AtomicRmwXor { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + loc, + target, + ret, + memarg, + 8, + Size::S64, + Size::S64, + |a, _m, src, dst| { + a.emit_xor(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I32AtomicRmw8XorU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + loc, + target, + ret, + memarg, + 1, + Size::S8, + Size::S32, + |a, _m, src, dst| { + a.emit_xor(Size::S32, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I32AtomicRmw16XorU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + loc, + target, + ret, + memarg, + 1, + Size::S16, + Size::S32, + |a, _m, src, dst| { + a.emit_xor(Size::S32, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I64AtomicRmw8XorU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + loc, + target, + ret, + memarg, + 1, + Size::S8, + Size::S64, + |a, _m, src, dst| { + a.emit_xor(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I64AtomicRmw16XorU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + loc, + target, + ret, + memarg, + 1, + Size::S16, + Size::S64, + |a, _m, src, dst| { + a.emit_xor(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I64AtomicRmw32XorU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + loc, + target, + ret, + memarg, + 1, + Size::S32, + Size::S64, + |a, _m, src, dst| { + a.emit_xor(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I32AtomicRmwXchg { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S32, loc, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 4, + |a, _m, addr| { + a.emit_xchg(Size::S32, Location::GPR(value), Location::Memory(addr, 0)); + Ok(()) + }, + )?; + a.emit_mov(Size::S32, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmwXchg { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S64, loc, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 8, + |a, _m, addr| { + a.emit_xchg(Size::S64, Location::GPR(value), Location::Memory(addr, 0)); + Ok(()) + }, + )?; + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I32AtomicRmw8XchgU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S8, loc, Size::S32, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 1, + |a, _m, addr| { + a.emit_xchg(Size::S8, Location::GPR(value), Location::Memory(addr, 0)); + Ok(()) + }, + )?; + a.emit_mov(Size::S32, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I32AtomicRmw16XchgU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S16, loc, Size::S32, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 2, + |a, _m, addr| { + a.emit_xchg(Size::S16, Location::GPR(value), Location::Memory(addr, 0)); + Ok(()) + }, + )?; + a.emit_mov(Size::S32, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmw8XchgU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S8, loc, Size::S64, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 1, + |a, _m, addr| { + a.emit_xchg(Size::S8, Location::GPR(value), Location::Memory(addr, 0)); + Ok(()) + }, + )?; + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmw16XchgU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S16, loc, Size::S64, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 2, + |a, _m, addr| { + a.emit_xchg(Size::S16, Location::GPR(value), Location::Memory(addr, 0)); + Ok(()) + }, + )?; + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmw32XchgU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S32, loc, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 4, + |a, _m, addr| { + a.emit_xchg(Size::S32, Location::GPR(value), Location::Memory(addr, 0)); + Ok(()) + }, + )?; + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I32AtomicRmwCmpxchg { ref memarg } => { + let new = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let cmp = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let compare = self.machine.reserve_unused_temp_gpr(GPR::RAX); + let value = if cmp == Location::GPR(GPR::R14) { + if new == Location::GPR(GPR::R13) { + GPR::R12 + } else { + GPR::R13 + } + } else { + GPR::R14 + }; + a.emit_push(Size::S64, Location::GPR(value)); + a.emit_mov(Size::S32, cmp, Location::GPR(compare)); + a.emit_mov(Size::S32, new, Location::GPR(value)); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 4, + |a, _m, addr| { + a.emit_lock_cmpxchg( + Size::S32, + Location::GPR(value), + Location::Memory(addr, 0), + ); + a.emit_mov(Size::S32, Location::GPR(compare), ret); + Ok(()) + }, + )?; + a.emit_pop(Size::S64, Location::GPR(value)); + self.machine.release_temp_gpr(compare); + } + Operator::I64AtomicRmwCmpxchg { ref memarg } => { + let new = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let cmp = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let compare = self.machine.reserve_unused_temp_gpr(GPR::RAX); + let value = if cmp == Location::GPR(GPR::R14) { + if new == Location::GPR(GPR::R13) { + GPR::R12 + } else { + GPR::R13 + } + } else { + GPR::R14 + }; + a.emit_push(Size::S64, Location::GPR(value)); + a.emit_mov(Size::S64, cmp, Location::GPR(compare)); + a.emit_mov(Size::S64, new, Location::GPR(value)); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 8, + |a, _m, addr| { + a.emit_lock_cmpxchg( + Size::S64, + Location::GPR(value), + Location::Memory(addr, 0), + ); + a.emit_mov(Size::S64, Location::GPR(compare), ret); + Ok(()) + }, + )?; + a.emit_pop(Size::S64, Location::GPR(value)); + self.machine.release_temp_gpr(compare); + } + Operator::I32AtomicRmw8CmpxchgU { ref memarg } => { + let new = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let cmp = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let compare = self.machine.reserve_unused_temp_gpr(GPR::RAX); + let value = if cmp == Location::GPR(GPR::R14) { + if new == Location::GPR(GPR::R13) { + GPR::R12 + } else { + GPR::R13 + } + } else { + GPR::R14 + }; + a.emit_push(Size::S64, Location::GPR(value)); + a.emit_mov(Size::S32, cmp, Location::GPR(compare)); + a.emit_mov(Size::S32, new, Location::GPR(value)); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 1, + |a, _m, addr| { + a.emit_lock_cmpxchg( + Size::S8, + Location::GPR(value), + Location::Memory(addr, 0), + ); + a.emit_movzx(Size::S8, Location::GPR(compare), Size::S32, ret); + Ok(()) + }, + )?; + a.emit_pop(Size::S64, Location::GPR(value)); + self.machine.release_temp_gpr(compare); + } + Operator::I32AtomicRmw16CmpxchgU { ref memarg } => { + let new = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let cmp = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let compare = self.machine.reserve_unused_temp_gpr(GPR::RAX); + let value = if cmp == Location::GPR(GPR::R14) { + if new == Location::GPR(GPR::R13) { + GPR::R12 + } else { + GPR::R13 + } + } else { + GPR::R14 + }; + a.emit_push(Size::S64, Location::GPR(value)); + a.emit_mov(Size::S32, cmp, Location::GPR(compare)); + a.emit_mov(Size::S32, new, Location::GPR(value)); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 1, + |a, _m, addr| { + a.emit_lock_cmpxchg( + Size::S16, + Location::GPR(value), + Location::Memory(addr, 0), + ); + a.emit_movzx(Size::S16, Location::GPR(compare), Size::S32, ret); + Ok(()) + }, + )?; + a.emit_pop(Size::S64, Location::GPR(value)); + self.machine.release_temp_gpr(compare); + } + Operator::I64AtomicRmw8CmpxchgU { ref memarg } => { + let new = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let cmp = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let compare = self.machine.reserve_unused_temp_gpr(GPR::RAX); + let value = if cmp == Location::GPR(GPR::R14) { + if new == Location::GPR(GPR::R13) { + GPR::R12 + } else { + GPR::R13 + } + } else { + GPR::R14 + }; + a.emit_push(Size::S64, Location::GPR(value)); + a.emit_mov(Size::S64, cmp, Location::GPR(compare)); + a.emit_mov(Size::S64, new, Location::GPR(value)); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 1, + |a, _m, addr| { + a.emit_lock_cmpxchg( + Size::S8, + Location::GPR(value), + Location::Memory(addr, 0), + ); + a.emit_movzx(Size::S8, Location::GPR(compare), Size::S64, ret); + Ok(()) + }, + )?; + a.emit_pop(Size::S64, Location::GPR(value)); + self.machine.release_temp_gpr(compare); + } + Operator::I64AtomicRmw16CmpxchgU { ref memarg } => { + let new = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let cmp = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let compare = self.machine.reserve_unused_temp_gpr(GPR::RAX); + let value = if cmp == Location::GPR(GPR::R14) { + if new == Location::GPR(GPR::R13) { + GPR::R12 + } else { + GPR::R13 + } + } else { + GPR::R14 + }; + a.emit_push(Size::S64, Location::GPR(value)); + a.emit_mov(Size::S64, cmp, Location::GPR(compare)); + a.emit_mov(Size::S64, new, Location::GPR(value)); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 1, + |a, _m, addr| { + a.emit_lock_cmpxchg( + Size::S16, + Location::GPR(value), + Location::Memory(addr, 0), + ); + a.emit_movzx(Size::S16, Location::GPR(compare), Size::S64, ret); + Ok(()) + }, + )?; + a.emit_pop(Size::S64, Location::GPR(value)); + self.machine.release_temp_gpr(compare); + } + Operator::I64AtomicRmw32CmpxchgU { ref memarg } => { + let new = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let cmp = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let compare = self.machine.reserve_unused_temp_gpr(GPR::RAX); + let value = if cmp == Location::GPR(GPR::R14) { + if new == Location::GPR(GPR::R13) { + GPR::R12 + } else { + GPR::R13 + } + } else { + GPR::R14 + }; + a.emit_push(Size::S64, Location::GPR(value)); + a.emit_mov(Size::S64, cmp, Location::GPR(compare)); + a.emit_mov(Size::S64, new, Location::GPR(value)); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + &mut self.exception_table, + target, + memarg, + true, + 1, + |a, _m, addr| { + a.emit_lock_cmpxchg( + Size::S32, + Location::GPR(value), + Location::Memory(addr, 0), + ); + a.emit_mov(Size::S32, Location::GPR(compare), ret); + Ok(()) + }, + )?; + a.emit_pop(Size::S64, Location::GPR(value)); + self.machine.release_temp_gpr(compare); + } + _ => { + return Err(CodegenError { + message: format!("not yet implemented: {:?}", op), + }); + } + } + + Ok(()) + } +} + +fn type_to_wp_type(ty: Type) -> WpType { + match ty { + Type::I32 => WpType::I32, + Type::I64 => WpType::I64, + Type::F32 => WpType::F32, + Type::F64 => WpType::F64, + Type::V128 => WpType::V128, + } +} + +fn get_location_released(a: &mut Assembler, m: &mut Machine, loc: Location) -> Location { + m.release_locations(a, &[loc]); + loc +} + +fn sort_call_movs(movs: &mut [(Location, GPR)]) { + for i in 0..movs.len() { + for j in (i + 1)..movs.len() { + if let Location::GPR(src_gpr) = movs[j].0 { + if src_gpr == movs[i].1 { + movs.swap(i, j); + } + } + } + } + + /* + { + use std::collections::{HashMap, HashSet, VecDeque}; + let mut mov_map: HashMap> = HashMap::new(); + for mov in movs.iter() { + if let Location::GPR(src_gpr) = mov.0 { + if src_gpr != mov.1 { + mov_map.entry(src_gpr).or_insert_with(|| HashSet::new()).insert(mov.1); + } + } + } + + for (start, _) in mov_map.iter() { + let mut q: VecDeque = VecDeque::new(); + let mut black: HashSet = HashSet::new(); + + q.push_back(*start); + black.insert(*start); + + while q.len() > 0 { + let reg = q.pop_front().unwrap(); + let empty_set = HashSet::new(); + for x in mov_map.get(®).unwrap_or(&empty_set).iter() { + if black.contains(x) { + panic!("cycle detected"); + } + q.push_back(*x); + black.insert(*x); + } + } + } + } + */ +} + +// Constants for the bounds of truncation operations. These are the least or +// greatest exact floats in either f32 or f64 representation less-than (for +// least) or greater-than (for greatest) the i32 or i64 or u32 or u64 +// min (for least) or max (for greatest), when rounding towards zero. + +/// Greatest Exact Float (32 bits) less-than i32::MIN when rounding towards zero. +const GEF32_LT_I32_MIN: f32 = -2147483904.0; +/// Least Exact Float (32 bits) greater-than i32::MAX when rounding towards zero. +const LEF32_GT_I32_MAX: f32 = 2147483648.0; +/// Greatest Exact Float (32 bits) less-than i64::MIN when rounding towards zero. +const GEF32_LT_I64_MIN: f32 = -9223373136366403584.0; +/// Least Exact Float (32 bits) greater-than i64::MAX when rounding towards zero. +const LEF32_GT_I64_MAX: f32 = 9223372036854775808.0; +/// Greatest Exact Float (32 bits) less-than u32::MIN when rounding towards zero. +const GEF32_LT_U32_MIN: f32 = -1.0; +/// Least Exact Float (32 bits) greater-than u32::MAX when rounding towards zero. +const LEF32_GT_U32_MAX: f32 = 4294967296.0; +/// Greatest Exact Float (32 bits) less-than u64::MIN when rounding towards zero. +const GEF32_LT_U64_MIN: f32 = -1.0; +/// Least Exact Float (32 bits) greater-than u64::MAX when rounding towards zero. +const LEF32_GT_U64_MAX: f32 = 18446744073709551616.0; + +/// Greatest Exact Float (64 bits) less-than i32::MIN when rounding towards zero. +const GEF64_LT_I32_MIN: f64 = -2147483649.0; +/// Least Exact Float (64 bits) greater-than i32::MAX when rounding towards zero. +const LEF64_GT_I32_MAX: f64 = 2147483648.0; +/// Greatest Exact Float (64 bits) less-than i64::MIN when rounding towards zero. +const GEF64_LT_I64_MIN: f64 = -9223372036854777856.0; +/// Least Exact Float (64 bits) greater-than i64::MAX when rounding towards zero. +const LEF64_GT_I64_MAX: f64 = 9223372036854775808.0; +/// Greatest Exact Float (64 bits) less-than u32::MIN when rounding towards zero. +const GEF64_LT_U32_MIN: f64 = -1.0; +/// Least Exact Float (64 bits) greater-than u32::MAX when rounding towards zero. +const LEF64_GT_U32_MAX: f64 = 4294967296.0; +/// Greatest Exact Float (64 bits) less-than u64::MIN when rounding towards zero. +const GEF64_LT_U64_MIN: f64 = -1.0; +/// Least Exact Float (64 bits) greater-than u64::MAX when rounding towards zero. +const LEF64_GT_U64_MAX: f64 = 18446744073709551616.0; diff --git a/lib/singlepass-backend/src/lib.rs b/lib/singlepass-backend/src/lib.rs new file mode 100644 index 000000000000..701c4c47e835 --- /dev/null +++ b/lib/singlepass-backend/src/lib.rs @@ -0,0 +1,55 @@ +#![deny( + nonstandard_style, + unused_imports, + unused_mut, + unused_variables, + unused_unsafe, + unreachable_patterns +)] +#![doc(html_favicon_url = "https://wasmer.io/static/icons/favicon.ico")] +#![doc(html_logo_url = "https://avatars3.githubusercontent.com/u/44205449?s=200&v=4")] + +#[cfg(not(any( + all(target_os = "freebsd", target_arch = "x86_64"), + all(target_os = "freebsd", target_arch = "aarch64"), + all(target_os = "macos", target_arch = "x86_64"), + all(target_os = "linux", target_arch = "x86_64"), + all(target_os = "linux", target_arch = "aarch64"), +)))] +compile_error!("This crate doesn't yet support compiling on operating systems other than FreeBSD, linux and macos and architectures other than x86_64"); + +extern crate dynasmrt; + +extern crate serde; + +#[macro_use] +extern crate serde_derive; + +#[macro_use] +extern crate dynasm; + +#[macro_use] +extern crate lazy_static; + +extern crate byteorder; +#[macro_use] +extern crate smallvec; + +mod codegen_x64; +mod emitter_x64; +mod machine; +#[cfg(target_arch = "aarch64")] +mod translator_aarch64; + +pub use codegen_x64::USE_RKYV_SERIALIZATION; + +pub use codegen_x64::X64FunctionCode as FunctionCodeGenerator; +pub use codegen_x64::X64ModuleCodeGenerator as ModuleCodeGenerator; + +use wasmer_runtime_core::codegen::SimpleStreamingCompilerGen; +pub type SinglePassCompiler = SimpleStreamingCompilerGen< + codegen_x64::X64ModuleCodeGenerator, + codegen_x64::X64FunctionCode, + codegen_x64::X64ExecutionContext, + codegen_x64::CodegenError, +>; diff --git a/lib/spectests/Cargo.toml b/lib/spectests/Cargo.toml new file mode 100644 index 000000000000..3e6a83975264 --- /dev/null +++ b/lib/spectests/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "wasmer-spectests" +version = "0.15.0" +description = "Wasmer spectests library" +license = "MIT" +authors = ["The Wasmer Engineering Team "] +repository = "https://github.com/ElrondNetwork/wasmer" +edition = "2018" + +[dependencies] +glob = "0.3" +wasmer-runtime = { path = "../runtime", version = "0.15.0", default-features = false} +wasmer-clif-backend = { path = "../clif-backend", version = "0.15.0", optional = true} +wasmer-llvm-backend = { path = "../llvm-backend", version = "0.15.0", features = ["test"], optional = true } +wasmer-singlepass-backend = { path = "../singlepass-backend", version = "0.15.0", optional = true } + +[build-dependencies] +wabt = "0.9.1" + +[dev-dependencies] +wabt = "0.9.1" + +[features] +default = ["fast-tests"] +fast-tests = [] +# clif = ["wasmer-clif-backend", "wasmer-runtime/default-backend-cranelift"] +singlepass = ["wasmer-singlepass-backend", "wasmer-runtime/default-backend-singlepass"] +llvm = ["wasmer-llvm-backend", "wasmer-runtime/default-backend-llvm"] diff --git a/lib/wasi-tests/Cargo.toml b/lib/wasi-tests/Cargo.toml new file mode 100644 index 000000000000..5ea9f4092d79 --- /dev/null +++ b/lib/wasi-tests/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "wasmer-wasi-tests" +version = "0.15.0" +description = "Tests for our WASI implementation" +license = "MIT" +authors = ["The Wasmer Engineering Team "] +edition = "2018" +publish = false +build = "build/mod.rs" + +[dependencies] +# We set default features to false to be able to use the singlepass backend properly +wasmer-runtime = { path = "../runtime", version = "0.15.0", default-features = false } +wasmer-wasi = { path = "../wasi", version = "0.15.0" } +# hack to get tests to work +wasmer-clif-backend = { path = "../clif-backend", version = "0.15.0", optional = true} +wasmer-singlepass-backend = { path = "../singlepass-backend", version = "0.15.0", optional = true } +wasmer-llvm-backend = { path = "../llvm-backend", version = "0.15.0", features = ["test"], optional = true } + +[build-dependencies] +glob = "0.3" + +[dev-dependencies] +wasmer-dev-utils = { path = "../dev-utils", version = "0.15.0"} + +[features] +# clif = ["wasmer-clif-backend", "wasmer-runtime/default-backend-cranelift"] +singlepass = ["wasmer-singlepass-backend", "wasmer-runtime/default-backend-singlepass"] +llvm = ["wasmer-llvm-backend", "wasmer-runtime/default-backend-llvm"] diff --git a/lib/wasi/Cargo.toml b/lib/wasi/Cargo.toml index 285b23b20ace..1d342b340503 100644 --- a/lib/wasi/Cargo.toml +++ b/lib/wasi/Cargo.toml @@ -5,7 +5,7 @@ description = "WASI implementation library for Wasmer WebAssembly runtime" categories = ["wasm", "os"] keywords = ["wasm", "webassembly", "wasi", "sandbox", "ABI"] authors = ["Wasmer Engineering Team "] -repository = "https://github.com/wasmerio/wasmer" +repository = "https://github.com/ElrondNetwork/wasmer" license = "MIT" readme = "README.md" edition = "2018" diff --git a/lib/win-exception-handler/Cargo.toml b/lib/win-exception-handler/Cargo.toml new file mode 100644 index 000000000000..841c635b4858 --- /dev/null +++ b/lib/win-exception-handler/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "wasmer-win-exception-handler" +version = "0.15.0" +description = "Wasmer runtime exception handling for Windows" +license = "MIT" +authors = ["The Wasmer Engineering Team "] +repository = "https://github.com/ElrondNetwork/wasmer" +edition = "2018" + +[target.'cfg(windows)'.dependencies] +wasmer-runtime-core = { path = "../runtime-core", version = "0.15.0" } +winapi = { version = "0.3.8", features = ["winbase", "errhandlingapi", "minwindef", "minwinbase", "winnt"] } +libc = "0.2.60" + +[build-dependencies] +cmake = "0.1" diff --git a/src/bin/wasmer.rs b/src/bin/wasmer.rs new file mode 100644 index 000000000000..16b087964dd8 --- /dev/null +++ b/src/bin/wasmer.rs @@ -0,0 +1,1179 @@ +#![deny( + dead_code, + nonstandard_style, + unused_imports, + unused_mut, + unused_variables, + unused_unsafe, + unreachable_patterns +)] +extern crate structopt; +#[macro_use] +extern crate log; + +use std::collections::HashMap; +use std::env; +use std::fs::{read_to_string, File}; +use std::io; +use std::io::Read; +use std::path::PathBuf; +use std::process::exit; +use std::str::FromStr; + +use structopt::{clap, StructOpt}; + +use wasmer::*; +#[cfg(feature = "backend-cranelift")] +use wasmer_clif_backend::CraneliftCompiler; +#[cfg(feature = "backend-llvm")] +use wasmer_llvm_backend::{ + InkwellMemoryBuffer, InkwellModule, LLVMBackendConfig, LLVMCallbacks, LLVMCompiler, +}; +use wasmer_runtime::{ + cache::{Cache as BaseCache, FileSystemCache, WasmHash}, + Backend, Value, VERSION, +}; +#[cfg(feature = "managed")] +use wasmer_runtime_core::tiering::{run_tiering, InteractiveShellContext, ShellExitOperation}; +use wasmer_runtime_core::{ + self, + backend::{Compiler, CompilerConfig, Features, MemoryBoundCheckMode}, + loader::{Instance as LoadedInstance, LocalLoader}, + Module, +}; +#[cfg(unix)] +use wasmer_runtime_core::{ + fault::{pop_code_version, push_code_version}, + state::CodeVersion, +}; +#[cfg(feature = "wasi")] +use wasmer_wasi; + +#[cfg(feature = "backend-llvm")] +use std::{cell::RefCell, io::Write, rc::Rc}; +#[cfg(feature = "backend-llvm")] +use wasmer_runtime_core::backend::BackendCompilerConfig; + +#[cfg(not(any( + feature = "backend-cranelift", + feature = "backend-llvm", + feature = "backend-singlepass" +)))] +compile_error!("Please enable one or more of the compiler backends"); + +#[derive(Debug, StructOpt)] +#[structopt(name = "wasmer", about = "Wasm execution runtime.", author)] +/// The options for the wasmer Command Line Interface +enum CLIOptions { + /// Run a WebAssembly file. Formats accepted: wasm, wat + #[structopt(name = "run")] + Run(Run), + + /// Wasmer cache + #[structopt(name = "cache")] + Cache(Cache), + + /// Validate a Web Assembly binary + #[structopt(name = "validate")] + Validate(Validate), + + /// Update wasmer to the latest version + #[structopt(name = "self-update")] + SelfUpdate, +} + +#[derive(Debug, StructOpt, Clone)] +struct PrestandardFeatures { + /// Enable support for the SIMD proposal. + #[structopt(long = "enable-simd")] + simd: bool, + + /// Enable support for the threads proposal. + #[structopt(long = "enable-threads")] + threads: bool, + + /// Enable support for all pre-standard proposals. + #[structopt(long = "enable-all")] + all: bool, +} + +impl PrestandardFeatures { + /// Generate [`wabt::Features`] struct from CLI options + #[cfg(feature = "wabt")] + pub fn into_wabt_features(&self) -> wabt::Features { + let mut features = wabt::Features::new(); + if self.simd || self.all { + features.enable_simd(); + } + if self.threads || self.all { + features.enable_threads(); + } + features.enable_sign_extension(); + features.enable_sat_float_to_int(); + features + } + + /// Generate [`Features`] struct from CLI options + pub fn into_backend_features(&self) -> Features { + Features { + simd: self.simd || self.all, + threads: self.threads || self.all, + } + } +} + +#[cfg(feature = "backend-llvm")] +#[derive(Debug, StructOpt, Clone)] +/// LLVM backend flags. +pub struct LLVMCLIOptions { + /// Emit LLVM IR before optimization pipeline. + #[structopt(long = "llvm-pre-opt-ir", parse(from_os_str))] + pre_opt_ir: Option, + + /// Emit LLVM IR after optimization pipeline. + #[structopt(long = "llvm-post-opt-ir", parse(from_os_str))] + post_opt_ir: Option, + + /// Emit LLVM generated native code object file. + #[structopt(long = "llvm-object-file", parse(from_os_str))] + obj_file: Option, +} + +#[derive(Debug, StructOpt, Clone)] +struct Run { + /// Disable the cache + #[structopt(long = "disable-cache")] + disable_cache: bool, + + /// Input file + #[structopt(parse(from_os_str))] + path: PathBuf, + + /// Name of the backend to use (x86_64) + #[cfg(target_arch = "x86_64")] + #[structopt( + long = "backend", + default_value = "auto", + case_insensitive = true, + possible_values = Backend::variants(), + )] + backend: Backend, + + /// Name of the backend to use (aarch64) + #[cfg(target_arch = "aarch64")] + #[structopt( + long = "backend", + default_value = "singlepass", + case_insensitive = true, + possible_values = Backend::variants(), + )] + backend: Backend, + + /// Invoke a specified function + #[structopt(long = "invoke", short = "i")] + invoke: Option, + + /// Emscripten symbol map + #[structopt(long = "em-symbol-map", parse(from_os_str), group = "emscripten")] + em_symbol_map: Option, + + /// Begin execution at the specified symbol + #[structopt(long = "em-entrypoint", group = "emscripten")] + em_entrypoint: Option, + + /// WASI pre-opened directory + #[structopt(long = "dir", multiple = true, group = "wasi")] + pre_opened_directories: Vec, + + /// Map a host directory to a different location for the wasm module + #[structopt(long = "mapdir", multiple = true)] + mapped_dirs: Vec, + + /// Pass custom environment variables + #[structopt(long = "env", multiple = true)] + env_vars: Vec, + + /// Custom code loader + #[structopt( + long = "loader", + case_insensitive = true, + possible_values = LoaderName::variants(), + )] + loader: Option, + + /// Path to previously saved instance image to resume. + #[cfg(feature = "managed")] + #[structopt(long = "resume")] + resume: Option, + + /// Optimized backends for higher tiers. + #[cfg(feature = "managed")] + #[structopt( + long = "optimized-backends", + multiple = true, + case_insensitive = true, + possible_values = Backend::variants(), + )] + optimized_backends: Vec, + + /// Whether or not state tracking should be disabled during compilation. + /// State tracking is necessary for tier switching and backtracing. + #[structopt(long = "track-state")] + track_state: bool, + + // Enable the CallTrace middleware. + #[structopt(long = "call-trace")] + call_trace: bool, + + // Enable the BlockTrace middleware. + #[structopt(long = "block-trace")] + block_trace: bool, + + /// The command name is a string that will override the first argument passed + /// to the wasm program. This is used in wapm to provide nicer output in + /// help commands and error messages of the running wasm program + #[structopt(long = "command-name", hidden = true)] + command_name: Option, + + /// A prehashed string, used to speed up start times by avoiding hashing the + /// wasm module. If the specified hash is not found, Wasmer will hash the module + /// as if no `cache-key` argument was passed. + #[structopt(long = "cache-key", hidden = true)] + cache_key: Option, + + #[cfg(feature = "backend-llvm")] + #[structopt(flatten)] + backend_llvm_options: LLVMCLIOptions, + + #[structopt(flatten)] + features: PrestandardFeatures, + + /// Enable non-standard experimental IO devices + #[cfg(feature = "experimental-io-devices")] + #[structopt(long = "enable-experimental-io-devices")] + enable_experimental_io_devices: bool, + + /// Enable debug output + #[cfg(feature = "debug")] + #[structopt(long = "debug", short = "d")] + debug: bool, + + /// Generate debug information for use in a debugger + #[structopt(long = "generate-debug-info", short = "g")] + generate_debug_info: bool, + + /// Application arguments + #[structopt(name = "--", multiple = true)] + args: Vec, +} + +impl Run { + /// Used with the `invoke` argument + fn parse_args(&self, module: &Module, fn_name: &str) -> Result, String> { + utils::parse_args(module, fn_name, &self.args) + .map_err(|e| format!("Invoke failed: {:?}", e)) + } +} + +#[allow(dead_code)] +#[derive(Debug, Copy, Clone)] +enum LoaderName { + Local, + #[cfg(feature = "loader-kernel")] + Kernel, +} + +impl LoaderName { + pub fn variants() -> &'static [&'static str] { + &[ + "local", + #[cfg(feature = "loader-kernel")] + "kernel", + ] + } +} + +impl FromStr for LoaderName { + type Err = String; + fn from_str(s: &str) -> Result { + match s.to_lowercase().as_str() { + "local" => Ok(LoaderName::Local), + #[cfg(feature = "loader-kernel")] + "kernel" => Ok(LoaderName::Kernel), + _ => Err(format!("The loader {} doesn't exist", s)), + } + } +} + +#[derive(Debug, StructOpt)] +enum Cache { + /// Clear the cache + #[structopt(name = "clean")] + Clean, + + /// Display the location of the cache + #[structopt(name = "dir")] + Dir, +} + +#[derive(Debug, StructOpt)] +struct Validate { + /// Input file + #[structopt(parse(from_os_str))] + path: PathBuf, + + #[structopt(flatten)] + features: PrestandardFeatures, +} + +/// Read the contents of a file +fn read_file_contents(path: &PathBuf) -> Result, io::Error> { + let mut buffer: Vec = Vec::new(); + let mut file = File::open(path)?; + file.read_to_end(&mut buffer)?; + // We force to close the file + drop(file); + Ok(buffer) +} + +fn get_cache_dir() -> PathBuf { + match env::var("WASMER_CACHE_DIR") { + Ok(dir) => { + let mut path = PathBuf::from(dir); + path.push(VERSION); + path + } + Err(_) => { + // We use a temporal directory for saving cache files + let mut temp_dir = env::temp_dir(); + temp_dir.push("wasmer"); + temp_dir.push(VERSION); + temp_dir + } + } +} + +fn get_mapped_dirs(input: &[String]) -> Result, String> { + let mut md = vec![]; + for entry in input.iter() { + if let [alias, real_dir] = entry.split(':').collect::>()[..] { + let pb = PathBuf::from(&real_dir); + if let Ok(pb_metadata) = pb.metadata() { + if !pb_metadata.is_dir() { + return Err(format!( + "\"{}\" exists, but it is not a directory", + &real_dir + )); + } + } else { + return Err(format!("Directory \"{}\" does not exist", &real_dir)); + } + md.push((alias.to_string(), pb)); + continue; + } + return Err(format!( + "Directory mappings must consist of two paths separate by a colon. Found {}", + &entry + )); + } + Ok(md) +} + +#[cfg(feature = "wasi")] +fn get_env_var_args(input: &[String]) -> Result, String> { + let mut ev = vec![]; + for entry in input.iter() { + if let [env_var, value] = entry.split('=').collect::>()[..] { + ev.push((env_var, value)); + } else { + return Err(format!( + "Env vars must be of the form =. Found {}", + &entry + )); + } + } + Ok(ev) +} + +/// Helper function for `execute_wasm` (the `Run` command) +#[cfg(feature = "wasi")] +fn execute_wasi( + wasi_version: wasmer_wasi::WasiVersion, + options: &Run, + env_vars: Vec<(&str, &str)>, + module: wasmer_runtime_core::Module, + mapped_dirs: Vec<(String, PathBuf)>, + _wasm_binary: &[u8], +) -> Result<(), String> { + let name = if let Some(cn) = &options.command_name { + cn.clone() + } else { + options.path.to_str().unwrap().to_owned() + }; + + let args = options.args.iter().cloned().map(|arg| arg.into_bytes()); + let preopened_files = options.pre_opened_directories.clone(); + let mut wasi_state_builder = wasmer_wasi::state::WasiState::new(&name); + wasi_state_builder + .args(args) + .envs(env_vars) + .preopen_dirs(preopened_files) + .map_err(|e| format!("Failed to preopen directories: {:?}", e))? + .map_dirs(mapped_dirs) + .map_err(|e| format!("Failed to preopen mapped directories: {:?}", e))?; + + #[cfg(feature = "experimental-io-devices")] + { + if options.enable_experimental_io_devices { + wasi_state_builder.setup_fs(Box::new(wasmer_wasi_experimental_io_devices::initialize)); + } + } + let wasi_state = wasi_state_builder.build().map_err(|e| format!("{:?}", e))?; + + let import_object = wasmer_wasi::generate_import_object_from_state(wasi_state, wasi_version); + + #[allow(unused_mut)] // mut used in feature + let mut instance = module + .instantiate(&import_object) + .map_err(|e| format!("Can't instantiate WASI module: {:?}", e))?; + + let start: wasmer_runtime::Func<(), ()> = + instance.func("_start").map_err(|e| format!("{:?}", e))?; + + #[cfg(feature = "managed")] + { + let start_raw: extern "C" fn(&mut wasmer_runtime_core::vm::Ctx) = + unsafe { ::std::mem::transmute(start.get_vm_func()) }; + + unsafe { + run_tiering( + module.info(), + &_wasm_binary, + if let Some(ref path) = options.resume { + let mut f = File::open(path).unwrap(); + let mut out: Vec = vec![]; + f.read_to_end(&mut out).unwrap(); + Some( + wasmer_runtime_core::state::InstanceImage::from_bytes(&out) + .map_err(|_| format!("failed to decode image"))?, + ) + } else { + None + }, + &import_object, + start_raw, + &mut instance, + options.backend.to_string(), + options + .optimized_backends + .iter() + .map( + |&backend| -> (Backend, Box Box + Send>) { + let options = options.clone(); + ( + backend.to_string(), + Box::new(move || { + get_compiler_by_backend(backend, &options).unwrap() + }), + ) + }, + ) + .collect(), + interactive_shell, + )? + }; + } + + #[cfg(not(feature = "managed"))] + { + let result; + + #[cfg(unix)] + let cv_pushed = if let Some(msm) = instance.module.runnable_module.get_module_state_map() { + push_code_version(CodeVersion { + baseline: true, + msm: msm, + base: instance.module.runnable_module.get_code().unwrap().as_ptr() as usize, + backend: options.backend.to_string(), + runnable_module: instance.module.runnable_module.clone(), + }); + true + } else { + false + }; + + if let Some(invoke_fn) = options.invoke.as_ref() { + eprintln!("WARNING: Invoking aribtrary functions with WASI is not officially supported in the WASI standard yet. Use this feature at your own risk!"); + let args = options.parse_args(&module, invoke_fn)?; + let invoke_result = instance + .dyn_func(invoke_fn) + .map_err(|e| format!("Invoke failed: {:?}", e))? + .call(&args) + .map_err(|e| format!("Calling invoke fn failed: {:?}", e))?; + println!("{}({:?}) returned {:?}", invoke_fn, args, invoke_result); + return Ok(()); + } else { + result = start.call(); + } + + #[cfg(unix)] + { + if cv_pushed { + pop_code_version().unwrap(); + } + } + + if let Err(ref err) = result { + if let Some(error_code) = err.0.downcast_ref::() { + std::process::exit(error_code.code as i32) + } + return Err(format!("error: {:?}", err)); + } + } + Ok(()) +} + +#[cfg(feature = "backend-llvm")] +impl LLVMCallbacks for LLVMCLIOptions { + fn preopt_ir_callback(&mut self, module: &InkwellModule) { + if let Some(filename) = &self.pre_opt_ir { + module.print_to_file(filename).unwrap(); + } + } + + fn postopt_ir_callback(&mut self, module: &InkwellModule) { + if let Some(filename) = &self.post_opt_ir { + module.print_to_file(filename).unwrap(); + } + } + + fn obj_memory_buffer_callback(&mut self, memory_buffer: &InkwellMemoryBuffer) { + if let Some(filename) = &self.obj_file { + let mem_buf_slice = memory_buffer.as_slice(); + let mut file = File::create(filename).unwrap(); + let mut pos = 0; + while pos < mem_buf_slice.len() { + pos += file.write(&mem_buf_slice[pos..]).unwrap(); + } + } + } +} + +/// Execute a wasm/wat file +fn execute_wasm(options: &Run) -> Result<(), String> { + if options.generate_debug_info && options.backend != Backend::Cranelift { + return Err("Generating debug information is currently only available with the `cranelift` backend.".to_owned()); + } + + let disable_cache = options.disable_cache; + + let mapped_dirs = get_mapped_dirs(&options.mapped_dirs[..])?; + #[cfg(feature = "wasi")] + let env_vars = get_env_var_args(&options.env_vars[..])?; + let wasm_path = &options.path; + + #[allow(unused_mut)] + let mut wasm_binary: Vec = read_file_contents(wasm_path).map_err(|err| { + format!( + "Can't read the file {}: {}", + wasm_path.as_os_str().to_string_lossy(), + err + ) + })?; + + let em_symbol_map = if let Some(em_symbol_map_path) = options.em_symbol_map.clone() { + let em_symbol_map_content: String = read_to_string(&em_symbol_map_path) + .map_err(|err| { + format!( + "Can't read symbol map file {}: {}", + em_symbol_map_path.as_os_str().to_string_lossy(), + err, + ) + })? + .to_owned(); + let mut em_symbol_map = HashMap::new(); + for line in em_symbol_map_content.lines() { + let mut split = line.split(':'); + let num_str = if let Some(ns) = split.next() { + ns + } else { + return Err( + "Can't parse symbol map (expected each entry to be of the form: `0:func_name`)" + .to_string(), + ); + }; + let num: u32 = num_str.parse::().map_err(|err| { + format!( + "Failed to parse {} as a number in symbol map: {}", + num_str, err + ) + })?; + let name_str: String = if let Some(name_str) = split.next() { + name_str + } else { + return Err( + "Can't parse symbol map (expected each entry to be of the form: `0:func_name`)" + .to_string(), + ); + } + .to_owned(); + + em_symbol_map.insert(num, name_str); + } + Some(em_symbol_map) + } else { + None + }; + + // Don't error on --enable-all for other backends. + if options.features.simd { + #[cfg(feature = "backend-llvm")] + { + if options.backend != Backend::LLVM { + return Err("SIMD is only supported in the LLVM backend for now".to_string()); + } + } + #[cfg(not(feature = "backend-llvm"))] + return Err("SIMD is not supported in this backend".to_string()); + } + + if !utils::is_wasm_binary(&wasm_binary) { + #[cfg(feature = "wabt")] + { + let features = options.features.into_wabt_features(); + wasm_binary = wabt::wat2wasm_with_features(wasm_binary, features).map_err(|e| { + format!( + "Can't convert from wast to wasm because \"{}\"{}", + e, + match e.kind() { + wabt::ErrorKind::Deserialize(s) + | wabt::ErrorKind::Parse(s) + | wabt::ErrorKind::ResolveNames(s) + | wabt::ErrorKind::Validate(s) => format!(":\n\n{}", s), + wabt::ErrorKind::Nul + | wabt::ErrorKind::WriteText + | wabt::ErrorKind::NonUtf8Result + | wabt::ErrorKind::WriteBinary => "".to_string(), + } + ) + })?; + } + + #[cfg(not(feature = "wabt"))] + { + return Err( + "Input is not a wasm binary and the `wabt` feature is not enabled".to_string(), + ); + } + } + + let compiler: Box = get_compiler_by_backend(options.backend, options) + .ok_or_else(|| { + format!( + "the requested backend, \"{}\", is not enabled", + options.backend.to_string() + ) + })?; + + #[allow(unused_mut)] + let mut backend_specific_config = None; + #[cfg(feature = "backend-llvm")] + { + if options.backend == Backend::LLVM { + backend_specific_config = Some(BackendCompilerConfig(Box::new(LLVMBackendConfig { + callbacks: Some(Rc::new(RefCell::new(options.backend_llvm_options.clone()))), + }))) + } + } + + let track_state = options.track_state; + + #[cfg(feature = "loader-kernel")] + let is_kernel_loader = if let Some(LoaderName::Kernel) = options.loader { + true + } else { + false + }; + + #[cfg(not(feature = "loader-kernel"))] + let is_kernel_loader = false; + + let module = if is_kernel_loader { + webassembly::compile_with_config_with( + &wasm_binary[..], + CompilerConfig { + symbol_map: em_symbol_map.clone(), + memory_bound_check_mode: MemoryBoundCheckMode::Disable, + enforce_stack_check: true, + + // Kernel loader does not support explicit preemption checkpoints. + full_preemption: false, + + track_state, + features: options.features.into_backend_features(), + backend_specific_config, + ..Default::default() + }, + &*compiler, + ) + .map_err(|e| format!("Can't compile module: {:?}", e))? + } else if disable_cache { + webassembly::compile_with_config_with( + &wasm_binary[..], + CompilerConfig { + symbol_map: em_symbol_map.clone(), + track_state, + + // Enable full preemption if state tracking is enabled. + // Preemption only makes sense with state information. + full_preemption: track_state, + + features: options.features.into_backend_features(), + backend_specific_config, + generate_debug_info: options.generate_debug_info, + ..Default::default() + }, + &*compiler, + ) + .map_err(|e| format!("Can't compile module: {:?}", e))? + } else { + // If we have cache enabled + let wasmer_cache_dir = get_cache_dir(); + + // We create a new cache instance. + // It could be possible to use any other kinds of caching, as long as they + // implement the Cache trait (with save and load functions) + let mut cache = unsafe { + FileSystemCache::new(wasmer_cache_dir).map_err(|e| format!("Cache error: {:?}", e))? + }; + let load_cache_key = || -> Result<_, String> { + if let Some(ref prehashed_cache_key) = options.cache_key { + if let Ok(module) = + WasmHash::decode(prehashed_cache_key).and_then(|prehashed_key| { + cache.load_with_backend(prehashed_key, options.backend) + }) + { + debug!("using prehashed key: {}", prehashed_cache_key); + return Ok(module); + } + } + // We generate a hash for the given binary, so we can use it as key + // for the Filesystem cache + let hash = WasmHash::generate(&wasm_binary); + + // cache.load will return the Module if it's able to deserialize it properly, and an error if: + // * The file is not found + // * The file exists, but it's corrupted or can't be converted to a module + match cache.load_with_backend(hash, options.backend) { + Ok(module) => { + // We are able to load the module from cache + Ok(module) + } + Err(_) => { + let module = webassembly::compile_with_config_with( + &wasm_binary[..], + CompilerConfig { + symbol_map: em_symbol_map.clone(), + track_state, + features: options.features.into_backend_features(), + backend_specific_config, + ..Default::default() + }, + &*compiler, + ) + .map_err(|e| format!("Can't compile module: {:?}", e))?; + // We try to save the module into a cache file + cache.store(hash, module.clone()).unwrap_or_default(); + + Ok(module) + } + } + }; + + load_cache_key()? + }; + + if let Some(loader) = options.loader { + let mut import_object = wasmer_runtime_core::import::ImportObject::new(); + import_object.allow_missing_functions = true; // Import initialization might be left to the loader. + let instance = module + .instantiate(&import_object) + .map_err(|e| format!("Can't instantiate loader module: {:?}", e))?; + + let mut args: Vec = Vec::new(); + for arg in options.args.iter() { + let x = arg.as_str().parse().map_err(|_| { + format!( + "Can't parse the provided argument {:?} as a integer", + arg.as_str() + ) + })?; + args.push(Value::I32(x)); + } + + let index = instance.resolve_func("_start").map_err(|_| { + format!("The loader requires a _start function to be present in the module") + })?; + + let mut ins: Box> = match loader { + LoaderName::Local => Box::new( + instance + .load(LocalLoader) + .map_err(|e| format!("Can't use the local loader: {:?}", e))?, + ), + #[cfg(feature = "loader-kernel")] + LoaderName::Kernel => Box::new( + instance + .load(::wasmer_kernel_loader::KernelLoader) + .map_err(|e| format!("Can't use the kernel loader: {:?}", e))?, + ), + }; + println!("{:?}", ins.call(index, &args)); + return Ok(()); + } + + // TODO: refactor this + if wasmer_emscripten::is_emscripten_module(&module) { + let mut emscripten_globals = wasmer_emscripten::EmscriptenGlobals::new(&module)?; + let import_object = wasmer_emscripten::generate_emscripten_env(&mut emscripten_globals); + let mut instance = module + .instantiate(&import_object) + .map_err(|e| format!("Can't instantiate emscripten module: {:?}", e))?; + + wasmer_emscripten::run_emscripten_instance( + &module, + &mut instance, + &mut emscripten_globals, + if let Some(cn) = &options.command_name { + cn + } else { + options.path.to_str().unwrap() + }, + options.args.iter().map(|arg| arg.as_str()).collect(), + options.em_entrypoint.clone(), + mapped_dirs, + ) + .map_err(|e| format!("{:?}", e))?; + } else { + #[cfg(feature = "wasi")] + let wasi_version = wasmer_wasi::get_wasi_version(&module, true); + #[cfg(feature = "wasi")] + let is_wasi = wasi_version.is_some(); + #[cfg(not(feature = "wasi"))] + let is_wasi = false; + + if is_wasi { + #[cfg(feature = "wasi")] + execute_wasi( + wasi_version.unwrap(), + options, + env_vars, + module, + mapped_dirs, + &wasm_binary, + )?; + } else { + let import_object = wasmer_runtime_core::import::ImportObject::new(); + let instance = module + .instantiate(&import_object) + .map_err(|e| format!("Can't instantiate module: {:?}", e))?; + + let invoke_fn = match options.invoke.as_ref() { + Some(fun) => fun, + _ => "main", + }; + let args = options.parse_args(&module, invoke_fn)?; + + #[cfg(unix)] + let cv_pushed = + if let Some(msm) = instance.module.runnable_module.get_module_state_map() { + push_code_version(CodeVersion { + baseline: true, + msm: msm, + base: instance.module.runnable_module.get_code().unwrap().as_ptr() as usize, + backend: options.backend.to_string(), + runnable_module: instance.module.runnable_module.clone(), + }); + true + } else { + false + }; + + let result = instance + .dyn_func(&invoke_fn) + .map_err(|e| format!("{:?}", e))? + .call(&args) + .map_err(|e| format!("{:?}", e))?; + + #[cfg(unix)] + { + if cv_pushed { + pop_code_version().unwrap(); + } + } + println!("{}({:?}) returned {:?}", invoke_fn, args, result); + } + } + + Ok(()) +} + +#[cfg(feature = "managed")] +fn interactive_shell(mut ctx: InteractiveShellContext) -> ShellExitOperation { + use std::io::Write; + + let mut stdout = ::std::io::stdout(); + let stdin = ::std::io::stdin(); + + loop { + print!("Wasmer> "); + stdout.flush().unwrap(); + let mut line = String::new(); + stdin.read_line(&mut line).unwrap(); + let mut parts = line.split(" ").filter(|x| x.len() > 0).map(|x| x.trim()); + + let cmd = parts.next(); + if cmd.is_none() { + println!("Command required"); + continue; + } + let cmd = cmd.unwrap(); + + match cmd { + "snapshot" => { + let path = parts.next(); + if path.is_none() { + println!("Usage: snapshot [out_path]"); + continue; + } + let path = path.unwrap(); + + if let Some(ref image) = ctx.image { + let buf = image.to_bytes(); + let mut f = match File::create(path) { + Ok(x) => x, + Err(e) => { + println!("Cannot open output file at {}: {:?}", path, e); + continue; + } + }; + if let Err(e) = f.write_all(&buf) { + println!("Cannot write to output file at {}: {:?}", path, e); + continue; + } + println!("Done"); + } else { + println!("Program state not available"); + } + } + "continue" | "c" => { + if let Some(image) = ctx.image.take() { + return ShellExitOperation::ContinueWith(image); + } else { + println!("Program state not available, cannot continue execution"); + } + } + "backtrace" | "bt" => { + if let Some(ref image) = ctx.image { + println!("{}", image.execution_state.output()); + } else { + println!("State not available"); + } + } + "exit" | "quit" => { + exit(0); + } + "" => {} + _ => { + println!("Unknown command: {}", cmd); + } + } + } +} + +#[allow(unused_variables, unreachable_code)] +fn get_backend(backend: Backend, path: &PathBuf) -> Backend { + // Update backend when a backend flag is `auto`. + // Use the Singlepass backend if it's enabled and the file provided is larger + // than 10MiB (10485760 bytes), or it's enabled and the target architecture + // is AArch64. Otherwise, use the Cranelift backend. + match backend { + Backend::Auto => { + #[cfg(feature = "backend-singlepass")] + { + let binary_size = match &path.metadata() { + Ok(wasm_binary) => wasm_binary.len(), + Err(_e) => 0, + }; + if binary_size > 10485760 || cfg!(target_arch = "aarch64") { + return Backend::Singlepass; + } + } + + #[cfg(feature = "backend-cranelift")] + { + return Backend::Cranelift; + } + + #[cfg(feature = "backend-llvm")] + { + return Backend::LLVM; + } + + panic!("Can't find any backend"); + } + backend => backend, + } +} + +fn run(options: &mut Run) { + options.backend = get_backend(options.backend, &options.path); + + #[cfg(any(feature = "debug", feature = "trace"))] + { + if options.debug { + logging::set_up_logging().expect("failed to set up logging"); + } + } + match execute_wasm(options) { + Ok(()) => {} + Err(message) => { + eprintln!("Error: {}", message); + exit(1); + } + } +} + +fn validate_wasm(validate: Validate) -> Result<(), String> { + let wasm_path = validate.path; + let wasm_path_as_str = wasm_path.to_str().unwrap(); + + let wasm_binary: Vec = read_file_contents(&wasm_path).map_err(|err| { + format!( + "Can't read the file {}: {}", + wasm_path.as_os_str().to_string_lossy(), + err + ) + })?; + + if !utils::is_wasm_binary(&wasm_binary) { + return Err(format!( + "Cannot recognize \"{}\" as a WASM binary", + wasm_path_as_str, + )); + } + + wasmer_runtime_core::validate_and_report_errors_with_features( + &wasm_binary, + validate.features.into_backend_features(), + ) + .map_err(|err| format!("Validation failed: {}", err))?; + + Ok(()) +} + +/// Runs logic for the `validate` subcommand +fn validate(validate: Validate) { + match validate_wasm(validate) { + Err(message) => { + eprintln!("Error: {}", message); + exit(-1); + } + _ => (), + } +} + +fn get_compiler_by_backend(backend: Backend, _opts: &Run) -> Option> { + Some(match backend { + #[cfg(feature = "backend-singlepass")] + Backend::Singlepass => { + use wasmer_runtime_core::codegen::MiddlewareChain; + use wasmer_runtime_core::codegen::StreamingCompiler; + use wasmer_singlepass_backend::ModuleCodeGenerator as SinglePassMCG; + + let opts = _opts.clone(); + let middlewares_gen = move || { + let mut middlewares = MiddlewareChain::new(); + if opts.call_trace { + use wasmer_middleware_common::call_trace::CallTrace; + middlewares.push(CallTrace::new()); + } + if opts.block_trace { + use wasmer_middleware_common::block_trace::BlockTrace; + middlewares.push(BlockTrace::new()); + } + middlewares + }; + + let c: StreamingCompiler = + StreamingCompiler::new(middlewares_gen); + Box::new(c) + } + #[cfg(not(feature = "backend-singlepass"))] + Backend::Singlepass => return None, + + #[cfg(feature = "backend-cranelift")] + Backend::Cranelift => Box::new(CraneliftCompiler::new()), + + #[cfg(not(feature = "backend-cranelift"))] + Backend::Cranelift => return None, + + #[cfg(feature = "backend-llvm")] + Backend::LLVM => Box::new(LLVMCompiler::new()), + + #[cfg(not(feature = "backend-llvm"))] + Backend::LLVM => return None, + Backend::Auto => return None, + _ => return None, + }) +} + +fn main() { + // We try to run wasmer with the normal arguments. + // Eg. `wasmer ` + // In case that fails, we fallback trying the Run subcommand directly. + // Eg. `wasmer myfile.wasm --dir=.` + let options = CLIOptions::from_iter_safe(env::args()).unwrap_or_else(|e| { + match e.kind { + // This fixes a issue that: + // 1. Shows the version twice when doing `wasmer -V` + // 2. Shows the run help (instead of normal help) when doing `wasmer --help` + clap::ErrorKind::VersionDisplayed | clap::ErrorKind::HelpDisplayed => e.exit(), + _ => CLIOptions::Run(Run::from_args()), + } + }); + match options { + CLIOptions::Run(mut options) => run(&mut options), + #[cfg(not(target_os = "windows"))] + CLIOptions::SelfUpdate => update::self_update(), + #[cfg(target_os = "windows")] + CLIOptions::SelfUpdate => { + println!("Self update is not supported on Windows. Use install instructions on the Wasmer homepage: https://wasmer.io"); + } + CLIOptions::Cache(cache) => match cache { + Cache::Clean => { + use std::fs; + let cache_dir = get_cache_dir(); + if cache_dir.exists() { + fs::remove_dir_all(cache_dir.clone()).expect("Can't remove cache dir"); + } + fs::create_dir_all(cache_dir.clone()).expect("Can't create cache dir"); + } + Cache::Dir => { + println!("{}", get_cache_dir().to_string_lossy()); + } + }, + CLIOptions::Validate(validate_options) => { + validate(validate_options); + } + } +} + +#[test] +fn filesystem_cache_should_work() -> Result<(), String> { + let wasmer_cache_dir = get_cache_dir(); + + unsafe { FileSystemCache::new(wasmer_cache_dir).map_err(|e| format!("Cache error: {:?}", e))? }; + + Ok(()) +} diff --git a/src/webassembly.rs b/src/webassembly.rs new file mode 100644 index 000000000000..0fd75b20ab41 --- /dev/null +++ b/src/webassembly.rs @@ -0,0 +1,71 @@ +pub use wasmer_runtime::compile_with_config_with; +use wasmer_runtime::{self as runtime, error::Result, ImportObject, Instance, Module}; + +pub struct ResultObject { + /// A webassembly::Module object representing the compiled WebAssembly module. + /// This Module can be instantiated again + pub module: Module, + /// A webassembly::Instance object that contains all the Exported WebAssembly + /// functions. + pub instance: Box, +} + +#[derive(PartialEq)] +pub enum InstanceABI { + Emscripten, + WASI, + None, +} + +/// The webassembly::instantiate() function allows you to compile and +/// instantiate WebAssembly code +/// Params: +/// * `buffer_source`: A `Vec` containing the +/// binary code of the .wasm module you want to compile. +/// * `import_object`: An object containing the values to be imported +/// into the newly-created Instance, such as functions or +/// webassembly::Memory objects. There must be one matching property +/// for each declared import of the compiled module or else a +/// webassembly::LinkError is thrown. +/// Errors: +/// If the operation fails, the Result rejects with a +/// webassembly::CompileError, webassembly::LinkError, or +/// webassembly::RuntimeError, depending on the cause of the failure. +pub fn instantiate(buffer_source: &[u8], import_object: ImportObject) -> Result { + debug!("webassembly - compiling module"); + let module = compile(&buffer_source[..])?; + + debug!("webassembly - instantiating"); + let instance = module.instantiate(&import_object)?; + + debug!("webassembly - instance created"); + Ok(ResultObject { + module, + instance: Box::new(instance), + }) +} + +/// The webassembly::instantiate_streaming() function compiles and instantiates +/// a WebAssembly module directly from a streamed underlying source. +/// This is the most efficient, optimized way to load wasm code. +pub fn instantiate_streaming( + _buffer_source: Vec, + _import_object: ImportObject, +) -> Result { + unimplemented!(); +} + +/// The webassembly::compile() function compiles a webassembly::Module +/// from WebAssembly binary code. This function is useful if it +/// is necessary to a compile a module before it can be instantiated +/// (otherwise, the webassembly::instantiate() function should be used). +/// Params: +/// * `buffer_source`: A `Vec` containing the +/// binary code of the .wasm module you want to compile. +/// Errors: +/// If the operation fails, the Result rejects with a +/// webassembly::CompileError. +pub fn compile(buffer_source: &[u8]) -> Result { + let module = runtime::compile(buffer_source)?; + Ok(module) +}