From b4d4a67e94607daa12477cabdd28f0ee095a509c Mon Sep 17 00:00:00 2001 From: Ben Heidemann Date: Sun, 2 Mar 2025 13:48:40 +0000 Subject: [PATCH 01/20] chore: move test cases to tests/fixtures directory --- CONTRIBUTING.md | 83 +++++++++++++++++++ .../validation_state.snap.json | 4 - .../validation_state.snap.json | 4 - .../validation_state.snap.json | 4 - .../validation_state.snap.json | 4 - .../validation_state.snap.json | 4 - .../validation_state.snap.json | 4 - .../validation_state.snap.json | 4 - .../validation_state.snap.json | 16 ---- .../validation_state.snap.json | 11 --- .../009_multi_file/validation_state.snap.json | 16 ---- .../fixtures}/001_basic_workflow/test.yml | 0 .../fixtures}/002_basic_action/action.yml | 0 .../fixtures}/003_successful_globs/glob.yml | 0 .../fixtures}/004_failing_globs/exitcode | 0 .../fixtures}/004_failing_globs/glob.yml | 0 .../fixtures}/004_failing_globs/stderr | 0 .../fixtures}/004_failing_globs/stdout | 0 .../004a_failing_negative_glob/exitcode | 0 .../004a_failing_negative_glob/glob.yml | 0 .../004a_failing_negative_glob/stderr | 0 .../004a_failing_negative_glob/stdout | 0 .../005_conditional_step_in_action/action.yml | 0 .../test.yml | 0 .../fixtures}/007_funky_syntax/exitcode | 0 .../fixtures}/007_funky_syntax/rust-check.yml | 0 .../fixtures}/007_funky_syntax/stderr | 0 .../fixtures}/007_funky_syntax/stdout | 0 .../fixtures}/008_job_dependencies/exitcode | 0 .../fixtures}/008_job_dependencies/stderr | 0 .../fixtures}/008_job_dependencies/stdout | 0 .../fixtures}/008_job_dependencies/test.yml | 0 .../fixtures}/009_multi_file/exitcode | 0 .../fixtures}/009_multi_file/stderr | 0 .../fixtures}/009_multi_file/stdout | 0 .../fixtures}/009_multi_file/valid.yml | 0 .../fixtures}/009_multi_file/xinvalid.yml | 0 .../010_missing_shell_in_action/action.yml | 0 .../010_missing_shell_in_action/exitcode | 0 .../010_missing_shell_in_action/stderr | 0 .../010_missing_shell_in_action/stdout | 0 41 files changed, 83 insertions(+), 71 deletions(-) delete mode 100644 test/001_basic_workflow/validation_state.snap.json delete mode 100644 test/002_basic_action/validation_state.snap.json delete mode 100644 test/003_successful_globs/validation_state.snap.json delete mode 100644 test/004_failing_globs/validation_state.snap.json delete mode 100644 test/004a_failing_negative_glob/validation_state.snap.json delete mode 100644 test/005_conditional_step_in_action/validation_state.snap.json delete mode 100644 test/006_workflow_dispatch_inputs_options/validation_state.snap.json delete mode 100644 test/007_funky_syntax/validation_state.snap.json delete mode 100644 test/008_job_dependencies/validation_state.snap.json delete mode 100644 test/009_multi_file/validation_state.snap.json rename {test => tests/fixtures}/001_basic_workflow/test.yml (100%) rename {test => tests/fixtures}/002_basic_action/action.yml (100%) rename {test => tests/fixtures}/003_successful_globs/glob.yml (100%) rename {test => tests/fixtures}/004_failing_globs/exitcode (100%) rename {test => tests/fixtures}/004_failing_globs/glob.yml (100%) rename {test => tests/fixtures}/004_failing_globs/stderr (100%) rename {test => tests/fixtures}/004_failing_globs/stdout (100%) rename {test => tests/fixtures}/004a_failing_negative_glob/exitcode (100%) rename {test => tests/fixtures}/004a_failing_negative_glob/glob.yml (100%) rename {test => tests/fixtures}/004a_failing_negative_glob/stderr (100%) rename {test => tests/fixtures}/004a_failing_negative_glob/stdout (100%) rename {test => tests/fixtures}/005_conditional_step_in_action/action.yml (100%) rename {test => tests/fixtures}/006_workflow_dispatch_inputs_options/test.yml (100%) rename {test => tests/fixtures}/007_funky_syntax/exitcode (100%) rename {test => tests/fixtures}/007_funky_syntax/rust-check.yml (100%) rename {test => tests/fixtures}/007_funky_syntax/stderr (100%) rename {test => tests/fixtures}/007_funky_syntax/stdout (100%) rename {test => tests/fixtures}/008_job_dependencies/exitcode (100%) rename {test => tests/fixtures}/008_job_dependencies/stderr (100%) rename {test => tests/fixtures}/008_job_dependencies/stdout (100%) rename {test => tests/fixtures}/008_job_dependencies/test.yml (100%) rename {test => tests/fixtures}/009_multi_file/exitcode (100%) rename {test => tests/fixtures}/009_multi_file/stderr (100%) rename {test => tests/fixtures}/009_multi_file/stdout (100%) rename {test => tests/fixtures}/009_multi_file/valid.yml (100%) rename {test => tests/fixtures}/009_multi_file/xinvalid.yml (100%) rename {test => tests/fixtures}/010_missing_shell_in_action/action.yml (100%) rename {test => tests/fixtures}/010_missing_shell_in_action/exitcode (100%) rename {test => tests/fixtures}/010_missing_shell_in_action/stderr (100%) rename {test => tests/fixtures}/010_missing_shell_in_action/stdout (100%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0c2b407..1593235 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,3 +1,5 @@ +# Overview + * If you have found a discrepancy in documented and observed behaviour, that is a bug. Feel free to [report it as an issue](https://github.com/mpalmer/action-validator/issues), providing @@ -8,3 +10,84 @@ request](https://github.com/mpalmer/action-validator/pulls). * At all times, abide by the Code of Conduct (CODE_OF_CONDUCT.md). + +--- + +# Environment Setup + +## Install Rust +Firstly, you'll need make any changes to the core functionality of this project. We recommend use `rustup`, on the recommendation of the rust team. You can find the installation instructions [here](https://www.rust-lang.org/tools/install). + +To confirm that rust is installed, run the `cargo` command. If you don't receive the help docs output, you may need to add rust to your shell rc file. + +## Git Submodule Setup +This repository uses [git submodules](https://git-scm.com/book/en/v2/Git-Tools-Submodules). Specifically for the use of [schemastore](https://github.com/SchemaStore/schemastore). + +To setup the git submodule after cloning this repo to your local, you'll want to run the following commands: +1. `git submodule init` +2. `git submodule update` + +It should look similar to the output below. + +```bash +❯ git submodule init +Submodule 'src/schemastore' (https://github.com/SchemaStore/schemastore) registered for path 'src +/schemastore' +❯ git submodule update +Cloning into '/Users/someuser/action-validator/src/schemastore'... +Submodule path 'src/schemastore': checked out 'd3e6ab7727380b214acbab05570fb09a3e5d2dfc' +``` + +At this point, you should be all set to `cargo run`! If you run into any issues here, please [create an issue](https://github.com/mpalmer/action-validator/issues/new/choose). + +# Running the Validator Locally + +## `cargo run [FILE] -- [OPTIONS]` +`cargo run` will create a _debug_ executable and run the project. If this is your first time running the command, cargo will compile the development binary with `cargo build`. This will install all of the dependencies and create the debug binary `action-validator` in the `/target/debug/` directory. `cargo run` will then invoke this binary after creation. + +One caveat if you're running with `cargo run`: if you want to supply the program with options, you need to use the `--` operator between `cargo run` and your provided options. This let's cargo know which flags are meant for cargo, and which are meant for the executable. + +## `cargo build` && `./target/debug/action-validator [OPTIONS]` +As discussed in the prior section, `cargo build` install dependencies (if they're not cached) and build the development binary. This binary can then be invoked directly by running `./target/debug/action-validator`. This does **not** require the use of the `--` operator in between the binary and any provided options. + +## Try It Yourself! + +Run the command `cargo run -- --help`. You should see an output similar to the following. +```bash +❯ cargo run -- --help + Finished dev [unoptimized + debuginfo] target(s) in 0.05s + Running `target/debug/action-validator --help` +A validator for GitHub Action and Workflow YAML files + +Usage: action-validator [OPTIONS] [path_to_action_yaml]... + +Arguments: + [path_to_action_yaml]... Input file + +Options: + -v, --verbose Be more verbose + -h, --help Print help information + -V, --version Print version information +``` + +# Writing Tests +All tests live in the `tests` directory. Currently, this project implements snapshot testing, +but that's not to say you couldn't write unit or integration tests with the current structure. +To run the tests, simply run `cargo test` from the root directory. If you want to test a specific +feature, you can add the `-F {feature}` flag (e.g. `cargo test -F remote-checks`). + +## Unit/Integration Tests +As of this writing, there are no unit or integration tests. If you are looking to write some, please +follow the directions in [this guide](https://doc.rust-lang.org/book/ch11-01-writing-tests.html). + +## Snapshot Tests +A snapshot test is performed when we execute the cli and capture `stdout`, `stderr`, and/or an exit code. +When the tests is run, the results of the test must exactly match those of the previous run. For this project, +the snapshot tests are named in the format `{next_id}_{whats_being_tested}` (e.g. `011_remote_checks_failure`). + +If you have made changes which will change the output of the program and cause snapshots to fail, you can run +`cargo test -F save-snapshots`. This feature causes the executed command to save the `stdout`, `stderr`, and/or +exit code to the specified testing directory. + +If you are writing a net new test, you will need to create the test directory with your workflow or action file. +Once you're done, you can save the results to that directy by running `cargo test -F save-snapshots`. diff --git a/test/001_basic_workflow/validation_state.snap.json b/test/001_basic_workflow/validation_state.snap.json deleted file mode 100644 index c313801..0000000 --- a/test/001_basic_workflow/validation_state.snap.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "actionType": "workflow", - "errors": [] -} diff --git a/test/002_basic_action/validation_state.snap.json b/test/002_basic_action/validation_state.snap.json deleted file mode 100644 index 3a0fe83..0000000 --- a/test/002_basic_action/validation_state.snap.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "actionType": "action", - "errors": [] -} diff --git a/test/003_successful_globs/validation_state.snap.json b/test/003_successful_globs/validation_state.snap.json deleted file mode 100644 index c313801..0000000 --- a/test/003_successful_globs/validation_state.snap.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "actionType": "workflow", - "errors": [] -} diff --git a/test/004_failing_globs/validation_state.snap.json b/test/004_failing_globs/validation_state.snap.json deleted file mode 100644 index c313801..0000000 --- a/test/004_failing_globs/validation_state.snap.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "actionType": "workflow", - "errors": [] -} diff --git a/test/004a_failing_negative_glob/validation_state.snap.json b/test/004a_failing_negative_glob/validation_state.snap.json deleted file mode 100644 index c313801..0000000 --- a/test/004a_failing_negative_glob/validation_state.snap.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "actionType": "workflow", - "errors": [] -} diff --git a/test/005_conditional_step_in_action/validation_state.snap.json b/test/005_conditional_step_in_action/validation_state.snap.json deleted file mode 100644 index 3a0fe83..0000000 --- a/test/005_conditional_step_in_action/validation_state.snap.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "actionType": "action", - "errors": [] -} diff --git a/test/006_workflow_dispatch_inputs_options/validation_state.snap.json b/test/006_workflow_dispatch_inputs_options/validation_state.snap.json deleted file mode 100644 index c313801..0000000 --- a/test/006_workflow_dispatch_inputs_options/validation_state.snap.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "actionType": "workflow", - "errors": [] -} diff --git a/test/007_funky_syntax/validation_state.snap.json b/test/007_funky_syntax/validation_state.snap.json deleted file mode 100644 index 0ff3efe..0000000 --- a/test/007_funky_syntax/validation_state.snap.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "actionType": "workflow", - "errors": [ - { - "code": "parse_error", - "detail": "mapping values are not allowed in this context at line 38 column 15", - "path": "", - "title": "Parse Error", - "location": { - "index": 1069, - "line": 38, - "column": 15 - } - } - ] -} diff --git a/test/008_job_dependencies/validation_state.snap.json b/test/008_job_dependencies/validation_state.snap.json deleted file mode 100644 index c504e36..0000000 --- a/test/008_job_dependencies/validation_state.snap.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "actionType": "workflow", - "errors": [ - { - "code": "unresolved_job", - "detail": "unresolved job asdf", - "path": "/jobs/build/needs", - "title": "Unresolved job" - } - ] -} diff --git a/test/009_multi_file/validation_state.snap.json b/test/009_multi_file/validation_state.snap.json deleted file mode 100644 index fb78ced..0000000 --- a/test/009_multi_file/validation_state.snap.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "actionType": "workflow", - "errors": [ - { - "code": "parse_error", - "detail": "mapping values are not allowed in this context at line 36 column 15", - "path": "", - "title": "Parse Error", - "location": { - "index": 1069, - "line": 36, - "column": 15 - } - } - ] -} diff --git a/test/001_basic_workflow/test.yml b/tests/fixtures/001_basic_workflow/test.yml similarity index 100% rename from test/001_basic_workflow/test.yml rename to tests/fixtures/001_basic_workflow/test.yml diff --git a/test/002_basic_action/action.yml b/tests/fixtures/002_basic_action/action.yml similarity index 100% rename from test/002_basic_action/action.yml rename to tests/fixtures/002_basic_action/action.yml diff --git a/test/003_successful_globs/glob.yml b/tests/fixtures/003_successful_globs/glob.yml similarity index 100% rename from test/003_successful_globs/glob.yml rename to tests/fixtures/003_successful_globs/glob.yml diff --git a/test/004_failing_globs/exitcode b/tests/fixtures/004_failing_globs/exitcode similarity index 100% rename from test/004_failing_globs/exitcode rename to tests/fixtures/004_failing_globs/exitcode diff --git a/test/004_failing_globs/glob.yml b/tests/fixtures/004_failing_globs/glob.yml similarity index 100% rename from test/004_failing_globs/glob.yml rename to tests/fixtures/004_failing_globs/glob.yml diff --git a/test/004_failing_globs/stderr b/tests/fixtures/004_failing_globs/stderr similarity index 100% rename from test/004_failing_globs/stderr rename to tests/fixtures/004_failing_globs/stderr diff --git a/test/004_failing_globs/stdout b/tests/fixtures/004_failing_globs/stdout similarity index 100% rename from test/004_failing_globs/stdout rename to tests/fixtures/004_failing_globs/stdout diff --git a/test/004a_failing_negative_glob/exitcode b/tests/fixtures/004a_failing_negative_glob/exitcode similarity index 100% rename from test/004a_failing_negative_glob/exitcode rename to tests/fixtures/004a_failing_negative_glob/exitcode diff --git a/test/004a_failing_negative_glob/glob.yml b/tests/fixtures/004a_failing_negative_glob/glob.yml similarity index 100% rename from test/004a_failing_negative_glob/glob.yml rename to tests/fixtures/004a_failing_negative_glob/glob.yml diff --git a/test/004a_failing_negative_glob/stderr b/tests/fixtures/004a_failing_negative_glob/stderr similarity index 100% rename from test/004a_failing_negative_glob/stderr rename to tests/fixtures/004a_failing_negative_glob/stderr diff --git a/test/004a_failing_negative_glob/stdout b/tests/fixtures/004a_failing_negative_glob/stdout similarity index 100% rename from test/004a_failing_negative_glob/stdout rename to tests/fixtures/004a_failing_negative_glob/stdout diff --git a/test/005_conditional_step_in_action/action.yml b/tests/fixtures/005_conditional_step_in_action/action.yml similarity index 100% rename from test/005_conditional_step_in_action/action.yml rename to tests/fixtures/005_conditional_step_in_action/action.yml diff --git a/test/006_workflow_dispatch_inputs_options/test.yml b/tests/fixtures/006_workflow_dispatch_inputs_options/test.yml similarity index 100% rename from test/006_workflow_dispatch_inputs_options/test.yml rename to tests/fixtures/006_workflow_dispatch_inputs_options/test.yml diff --git a/test/007_funky_syntax/exitcode b/tests/fixtures/007_funky_syntax/exitcode similarity index 100% rename from test/007_funky_syntax/exitcode rename to tests/fixtures/007_funky_syntax/exitcode diff --git a/test/007_funky_syntax/rust-check.yml b/tests/fixtures/007_funky_syntax/rust-check.yml similarity index 100% rename from test/007_funky_syntax/rust-check.yml rename to tests/fixtures/007_funky_syntax/rust-check.yml diff --git a/test/007_funky_syntax/stderr b/tests/fixtures/007_funky_syntax/stderr similarity index 100% rename from test/007_funky_syntax/stderr rename to tests/fixtures/007_funky_syntax/stderr diff --git a/test/007_funky_syntax/stdout b/tests/fixtures/007_funky_syntax/stdout similarity index 100% rename from test/007_funky_syntax/stdout rename to tests/fixtures/007_funky_syntax/stdout diff --git a/test/008_job_dependencies/exitcode b/tests/fixtures/008_job_dependencies/exitcode similarity index 100% rename from test/008_job_dependencies/exitcode rename to tests/fixtures/008_job_dependencies/exitcode diff --git a/test/008_job_dependencies/stderr b/tests/fixtures/008_job_dependencies/stderr similarity index 100% rename from test/008_job_dependencies/stderr rename to tests/fixtures/008_job_dependencies/stderr diff --git a/test/008_job_dependencies/stdout b/tests/fixtures/008_job_dependencies/stdout similarity index 100% rename from test/008_job_dependencies/stdout rename to tests/fixtures/008_job_dependencies/stdout diff --git a/test/008_job_dependencies/test.yml b/tests/fixtures/008_job_dependencies/test.yml similarity index 100% rename from test/008_job_dependencies/test.yml rename to tests/fixtures/008_job_dependencies/test.yml diff --git a/test/009_multi_file/exitcode b/tests/fixtures/009_multi_file/exitcode similarity index 100% rename from test/009_multi_file/exitcode rename to tests/fixtures/009_multi_file/exitcode diff --git a/test/009_multi_file/stderr b/tests/fixtures/009_multi_file/stderr similarity index 100% rename from test/009_multi_file/stderr rename to tests/fixtures/009_multi_file/stderr diff --git a/test/009_multi_file/stdout b/tests/fixtures/009_multi_file/stdout similarity index 100% rename from test/009_multi_file/stdout rename to tests/fixtures/009_multi_file/stdout diff --git a/test/009_multi_file/valid.yml b/tests/fixtures/009_multi_file/valid.yml similarity index 100% rename from test/009_multi_file/valid.yml rename to tests/fixtures/009_multi_file/valid.yml diff --git a/test/009_multi_file/xinvalid.yml b/tests/fixtures/009_multi_file/xinvalid.yml similarity index 100% rename from test/009_multi_file/xinvalid.yml rename to tests/fixtures/009_multi_file/xinvalid.yml diff --git a/test/010_missing_shell_in_action/action.yml b/tests/fixtures/010_missing_shell_in_action/action.yml similarity index 100% rename from test/010_missing_shell_in_action/action.yml rename to tests/fixtures/010_missing_shell_in_action/action.yml diff --git a/test/010_missing_shell_in_action/exitcode b/tests/fixtures/010_missing_shell_in_action/exitcode similarity index 100% rename from test/010_missing_shell_in_action/exitcode rename to tests/fixtures/010_missing_shell_in_action/exitcode diff --git a/test/010_missing_shell_in_action/stderr b/tests/fixtures/010_missing_shell_in_action/stderr similarity index 100% rename from test/010_missing_shell_in_action/stderr rename to tests/fixtures/010_missing_shell_in_action/stderr diff --git a/test/010_missing_shell_in_action/stdout b/tests/fixtures/010_missing_shell_in_action/stdout similarity index 100% rename from test/010_missing_shell_in_action/stdout rename to tests/fixtures/010_missing_shell_in_action/stdout From 0424016a59454e3635e4e727522437e1cded3023 Mon Sep 17 00:00:00 2001 From: Ben Heidemann Date: Sun, 2 Mar 2025 14:04:19 +0000 Subject: [PATCH 02/20] test: add snapshot tests in rust --- tests/snapshot_tests.rs | 89 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 tests/snapshot_tests.rs diff --git a/tests/snapshot_tests.rs b/tests/snapshot_tests.rs new file mode 100644 index 0000000..6af0e6a --- /dev/null +++ b/tests/snapshot_tests.rs @@ -0,0 +1,89 @@ +use std::path::{Path, PathBuf}; +use std::{ffi::OsStr, fs}; + +use assert_cmd::Command; +use fixtures::fixtures; + +#[derive(Debug)] +struct SnapshotTest { + test_dir: PathBuf, + workflow_files: Vec, +} + +impl SnapshotTest { + fn new(test_dir: &Path) -> Self { + let workflow_files = fs::read_dir(test_dir) + .unwrap() + .filter_map(Result::ok) + .filter(|f| f.path().extension() == Some(OsStr::new("yml"))) + .map(|f| f.path().to_path_buf()) + .collect(); + + SnapshotTest { + test_dir: test_dir.to_path_buf(), + workflow_files, + } + } + + #[cfg(not(feature = "save-snapshots"))] + fn execute(self) { + let stderr = fs::read_to_string(self.test_dir.join("stderr")).unwrap_or(String::from("")); + + let stdout = fs::read_to_string(self.test_dir.join("stdout")).unwrap_or(String::from("")); + + let exitcode: i32 = fs::read_to_string(self.test_dir.join("exitcode")) + .map(|s| { + s.strip_suffix("\n") + .unwrap_or(s.as_str()) + .parse::() + .unwrap_or(0) + }) + .unwrap_or(0); + + Command::cargo_bin(env!("CARGO_PKG_NAME")) + .expect("binary to execute") + .args(self.workflow_files) + .assert() + .stdout(stdout) + .stderr(stderr) + .code(exitcode); + } + + #[cfg(feature = "save-snapshots")] + fn execute(&self) { + use std::fs::File; + use std::io::prelude::*; + + let result = Command::cargo_bin(env!("CARGO_PKG_NAME")) + .expect("binary to execute") + .args(&self.workflow_files) + .ok() + .unwrap_or_else(|e| e.as_output().unwrap().to_owned()); + + if !result.stdout.is_empty() { + File::create(self.test_dir.join("stdout")) + .unwrap() + .write_all(&result.stdout) + .unwrap(); + } + if !result.stderr.is_empty() { + File::create(self.test_dir.join("stderr")) + .unwrap() + .write_all(&result.stderr) + .unwrap(); + } + if let Some(exitcode) = result.status.code() { + if exitcode > 0 { + File::create(self.test_dir.join("exitcode")) + .unwrap() + .write_all(exitcode.to_string().as_bytes()) + .unwrap(); + } + } + } +} + +#[fixtures("tests/fixtures/*")] +fn snapshot(dir: &Path) { + SnapshotTest::new(dir).execute(); +} From bd1b0dbe016495e8fd7dadfb724fcbde2c4793d6 Mon Sep 17 00:00:00 2001 From: Ben Heidemann Date: Sun, 2 Mar 2025 14:04:29 +0000 Subject: [PATCH 03/20] test: update fixtures --- tests/fixtures/003_successful_globs/excluded_path.json | 1 + tests/fixtures/003_successful_globs/glob.yml | 4 ++-- tests/fixtures/004_failing_globs/glob.yml | 2 +- tests/fixtures/004_failing_globs/stderr | 4 ++-- tests/fixtures/004_failing_globs/stdout | 2 +- tests/fixtures/004a_failing_negative_glob/glob.yml | 2 +- tests/fixtures/004a_failing_negative_glob/stderr | 4 ++-- tests/fixtures/004a_failing_negative_glob/stdout | 2 +- tests/fixtures/007_funky_syntax/stderr | 2 +- tests/fixtures/007_funky_syntax/stdout | 2 +- tests/fixtures/008_job_dependencies/stderr | 2 +- tests/fixtures/008_job_dependencies/stdout | 2 +- tests/fixtures/009_multi_file/stderr | 2 +- tests/fixtures/009_multi_file/stdout | 2 +- tests/fixtures/010_missing_shell_in_action/stderr | 2 +- tests/fixtures/010_missing_shell_in_action/stdout | 2 +- 16 files changed, 19 insertions(+), 18 deletions(-) create mode 100644 tests/fixtures/003_successful_globs/excluded_path.json diff --git a/tests/fixtures/003_successful_globs/excluded_path.json b/tests/fixtures/003_successful_globs/excluded_path.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/tests/fixtures/003_successful_globs/excluded_path.json @@ -0,0 +1 @@ +{} diff --git a/tests/fixtures/003_successful_globs/glob.yml b/tests/fixtures/003_successful_globs/glob.yml index 2ba9749..e843dfa 100644 --- a/tests/fixtures/003_successful_globs/glob.yml +++ b/tests/fixtures/003_successful_globs/glob.yml @@ -3,8 +3,8 @@ name: Test on: push: paths: - - 003_successful_globs/* - - '!003_successful_globs/*.json' + - tests/fixtures/003_successful_globs/* + - '!tests/fixtures/003_successful_globs/*.json' defaults: run: diff --git a/tests/fixtures/004_failing_globs/glob.yml b/tests/fixtures/004_failing_globs/glob.yml index 7c4d27e..dd9fb7a 100644 --- a/tests/fixtures/004_failing_globs/glob.yml +++ b/tests/fixtures/004_failing_globs/glob.yml @@ -3,7 +3,7 @@ name: Bad globs, no biscuit on: push: paths: - - 004_bad_globs/*.txt + - tests/fixtures/004_failing_globs/*.txt defaults: run: diff --git a/tests/fixtures/004_failing_globs/stderr b/tests/fixtures/004_failing_globs/stderr index 2171874..c269377 100644 --- a/tests/fixtures/004_failing_globs/stderr +++ b/tests/fixtures/004_failing_globs/stderr @@ -3,13 +3,13 @@ Validation failed: ValidationState { Workflow, ), file_path: Some( - "004_failing_globs/glob.yml", + "tests/fixtures/004_failing_globs/glob.yml", ), errors: [ NoFilesMatchingGlob { code: "glob_not_matched", detail: Some( - "Glob \"004_bad_globs/*.txt\" in /on/push/paths does not match any files", + "Glob \"tests/fixtures/004_failing_globs/*.txt\" in /on/push/paths does not match any files", ), path: "/on/push/paths", title: "Glob does not match any files", diff --git a/tests/fixtures/004_failing_globs/stdout b/tests/fixtures/004_failing_globs/stdout index 4179a2f..5260569 100644 --- a/tests/fixtures/004_failing_globs/stdout +++ b/tests/fixtures/004_failing_globs/stdout @@ -1 +1 @@ -Fatal error validating 004_failing_globs/glob.yml +Fatal error validating tests/fixtures/004_failing_globs/glob.yml diff --git a/tests/fixtures/004a_failing_negative_glob/glob.yml b/tests/fixtures/004a_failing_negative_glob/glob.yml index 4cf6acc..577e41c 100644 --- a/tests/fixtures/004a_failing_negative_glob/glob.yml +++ b/tests/fixtures/004a_failing_negative_glob/glob.yml @@ -3,7 +3,7 @@ name: Bad globs, no biscuit on: push: paths: - - '!004a_failing_negative_glob/*.txt' + - '!tests/fixtures/004a_failing_negative_glob/*.txt' defaults: run: diff --git a/tests/fixtures/004a_failing_negative_glob/stderr b/tests/fixtures/004a_failing_negative_glob/stderr index 029517a..5ec3f5b 100644 --- a/tests/fixtures/004a_failing_negative_glob/stderr +++ b/tests/fixtures/004a_failing_negative_glob/stderr @@ -3,13 +3,13 @@ Validation failed: ValidationState { Workflow, ), file_path: Some( - "004a_failing_negative_glob/glob.yml", + "tests/fixtures/004a_failing_negative_glob/glob.yml", ), errors: [ NoFilesMatchingGlob { code: "glob_not_matched", detail: Some( - "Glob \"!004a_failing_negative_glob/*.txt\" in /on/push/paths does not match any files", + "Glob \"!tests/fixtures/004a_failing_negative_glob/*.txt\" in /on/push/paths does not match any files", ), path: "/on/push/paths", title: "Glob does not match any files", diff --git a/tests/fixtures/004a_failing_negative_glob/stdout b/tests/fixtures/004a_failing_negative_glob/stdout index 8c3b53a..ebdfc05 100644 --- a/tests/fixtures/004a_failing_negative_glob/stdout +++ b/tests/fixtures/004a_failing_negative_glob/stdout @@ -1 +1 @@ -Fatal error validating 004a_failing_negative_glob/glob.yml +Fatal error validating tests/fixtures/004a_failing_negative_glob/glob.yml diff --git a/tests/fixtures/007_funky_syntax/stderr b/tests/fixtures/007_funky_syntax/stderr index 17c1d62..e783831 100644 --- a/tests/fixtures/007_funky_syntax/stderr +++ b/tests/fixtures/007_funky_syntax/stderr @@ -3,7 +3,7 @@ Validation failed: ValidationState { Workflow, ), file_path: Some( - "007_funky_syntax/rust-check.yml", + "tests/fixtures/007_funky_syntax/rust-check.yml", ), errors: [ Parse { diff --git a/tests/fixtures/007_funky_syntax/stdout b/tests/fixtures/007_funky_syntax/stdout index f57da30..e3eb6d9 100644 --- a/tests/fixtures/007_funky_syntax/stdout +++ b/tests/fixtures/007_funky_syntax/stdout @@ -1 +1 @@ -Fatal error validating 007_funky_syntax/rust-check.yml +Fatal error validating tests/fixtures/007_funky_syntax/rust-check.yml diff --git a/tests/fixtures/008_job_dependencies/stderr b/tests/fixtures/008_job_dependencies/stderr index a8d64bf..d84bccb 100644 --- a/tests/fixtures/008_job_dependencies/stderr +++ b/tests/fixtures/008_job_dependencies/stderr @@ -3,7 +3,7 @@ Validation failed: ValidationState { Workflow, ), file_path: Some( - "008_job_dependencies/test.yml", + "tests/fixtures/008_job_dependencies/test.yml", ), errors: [ UnresolvedJob { diff --git a/tests/fixtures/008_job_dependencies/stdout b/tests/fixtures/008_job_dependencies/stdout index 73dc3ee..508bdcd 100644 --- a/tests/fixtures/008_job_dependencies/stdout +++ b/tests/fixtures/008_job_dependencies/stdout @@ -1 +1 @@ -Fatal error validating 008_job_dependencies/test.yml +Fatal error validating tests/fixtures/008_job_dependencies/test.yml diff --git a/tests/fixtures/009_multi_file/stderr b/tests/fixtures/009_multi_file/stderr index b4fe130..88f6b91 100644 --- a/tests/fixtures/009_multi_file/stderr +++ b/tests/fixtures/009_multi_file/stderr @@ -3,7 +3,7 @@ Validation failed: ValidationState { Workflow, ), file_path: Some( - "009_multi_file/xinvalid.yml", + "tests/fixtures/009_multi_file/xinvalid.yml", ), errors: [ Parse { diff --git a/tests/fixtures/009_multi_file/stdout b/tests/fixtures/009_multi_file/stdout index ec1a3fd..f65cb42 100644 --- a/tests/fixtures/009_multi_file/stdout +++ b/tests/fixtures/009_multi_file/stdout @@ -1 +1 @@ -Fatal error validating 009_multi_file/xinvalid.yml +Fatal error validating tests/fixtures/009_multi_file/xinvalid.yml diff --git a/tests/fixtures/010_missing_shell_in_action/stderr b/tests/fixtures/010_missing_shell_in_action/stderr index 84e3397..1c3a021 100644 --- a/tests/fixtures/010_missing_shell_in_action/stderr +++ b/tests/fixtures/010_missing_shell_in_action/stderr @@ -3,7 +3,7 @@ Validation failed: ValidationState { Action, ), file_path: Some( - "010_missing_shell_in_action/action.yml", + "tests/fixtures/010_missing_shell_in_action/action.yml", ), errors: [ OneOf { diff --git a/tests/fixtures/010_missing_shell_in_action/stdout b/tests/fixtures/010_missing_shell_in_action/stdout index 93c3cda..8e52206 100644 --- a/tests/fixtures/010_missing_shell_in_action/stdout +++ b/tests/fixtures/010_missing_shell_in_action/stdout @@ -1 +1 @@ -Fatal error validating 010_missing_shell_in_action/action.yml +Fatal error validating tests/fixtures/010_missing_shell_in_action/action.yml From b6ce89b259c42ee559634cb6f3af19dd5aec6642 Mon Sep 17 00:00:00 2001 From: Ben Heidemann Date: Sun, 2 Mar 2025 14:05:30 +0000 Subject: [PATCH 04/20] chore: take dependency on assert_cmd and fixtures --- Cargo.lock | 95 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 3 ++ 2 files changed, 98 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 8b440ed..150214c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6,8 +6,10 @@ version = 3 name = "action-validator" version = "0.0.0-git" dependencies = [ + "assert_cmd", "clap", "console_error_panic_hook", + "fixtures", "glob", "is-terminal", "js-sys", @@ -103,6 +105,22 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "assert_cmd" +version = "2.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1835b7f27878de8525dc71410b5a31cdcc5f230aed5ba5df968e09c201b23d" +dependencies = [ + "anstyle", + "bstr", + "doc-comment", + "libc", + "predicates", + "predicates-core", + "predicates-tree", + "wait-timeout", +] + [[package]] name = "autocfg" version = "1.4.0" @@ -130,6 +148,17 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" +[[package]] +name = "bstr" +version = "1.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "531a9155a481e2ee699d4f98f43c0ca4ff8ee1bfd55c31e9e98fb29d2b176fe0" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + [[package]] name = "bumpalo" version = "3.16.0" @@ -225,6 +254,12 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + [[package]] name = "displaydoc" version = "0.2.5" @@ -236,6 +271,12 @@ dependencies = [ "syn", ] +[[package]] +name = "doc-comment" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + [[package]] name = "downcast-rs" version = "1.2.1" @@ -267,6 +308,18 @@ dependencies = [ "regex", ] +[[package]] +name = "fixtures" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee641ed7294dab290809e5a675dae1d3a48fa31e62b94974b25490194f3a96a" +dependencies = [ + "glob", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -630,6 +683,33 @@ dependencies = [ "siphasher", ] +[[package]] +name = "predicates" +version = "3.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d19ee57562043d37e82899fade9a22ebab7be9cef5026b07fda9cdd4293573" +dependencies = [ + "anstyle", + "difflib", + "predicates-core", +] + +[[package]] +name = "predicates-core" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "727e462b119fe9c93fd0eb1429a5f7647394014cf3c04ab2c0350eeb09095ffa" + +[[package]] +name = "predicates-tree" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72dd2d6d381dfb73a193c7fca536518d7caee39fc8503f74e7dc0be0531b425c" +dependencies = [ + "predicates-core", + "termtree", +] + [[package]] name = "proc-macro2" version = "1.0.92" @@ -836,6 +916,12 @@ dependencies = [ "syn", ] +[[package]] +name = "termtree" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" + [[package]] name = "tinystr" version = "0.7.6" @@ -930,6 +1016,15 @@ dependencies = [ "uuid", ] +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + [[package]] name = "walkdir" version = "2.5.0" diff --git a/Cargo.toml b/Cargo.toml index f6ed7c8..96dde21 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ crate-type = ["cdylib", "rlib"] [features] js = ["console_error_panic_hook", "valico/js"] +save-snapshots = [] [dependencies] clap = { version = "4.0", features = ["derive"] } @@ -42,6 +43,8 @@ console_error_panic_hook = { version = "0.1.7", optional = true } serde-wasm-bindgen = "0.4.5" js-sys = "0.3.77" is-terminal = "0.4.7" +fixtures = "0.1.0" +assert_cmd = "2.0.8" [dev-dependencies] wasm-bindgen-test = "0.3.34" From beda0c7469778149e2ff66c1426c59f3d40b43a6 Mon Sep 17 00:00:00 2001 From: Ben Heidemann Date: Sun, 2 Mar 2025 15:04:58 +0000 Subject: [PATCH 05/20] test: allow running tests against node/WASM bindings --- Cargo.toml | 6 ++++-- package-lock.json | 39 +++++++++++++++++++++++++++++++++++++++ package.json | 6 ++++-- scripts/build-wasm.sh | 4 +++- scripts/test-wasm.sh | 6 ++++++ tests/snapshot_tests.rs | 14 ++++++++++++-- 6 files changed, 68 insertions(+), 7 deletions(-) create mode 100644 scripts/test-wasm.sh diff --git a/Cargo.toml b/Cargo.toml index 96dde21..a54f635 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,8 +20,10 @@ rust-version = "1.60.0" crate-type = ["cdylib", "rlib"] [features] +default = ["assert_cmd"] js = ["console_error_panic_hook", "valico/js"] -save-snapshots = [] +test-save-snapshots = [] +test-js = [] [dependencies] clap = { version = "4.0", features = ["derive"] } @@ -44,7 +46,7 @@ serde-wasm-bindgen = "0.4.5" js-sys = "0.3.77" is-terminal = "0.4.7" fixtures = "0.1.0" -assert_cmd = "2.0.8" +assert_cmd = { version = "2.0.8", optional = true } [dev-dependencies] wasm-bindgen-test = "0.3.34" diff --git a/package-lock.json b/package-lock.json index 23fef40..b40b52e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,9 @@ "name": "@action-validator/root", "version": "0.0.0-git", "license": "GPL-3.0-only", + "workspaces": [ + "packages/*" + ], "devDependencies": { "chalk": "5.2.0", "diff": "5.1.0", @@ -15,6 +18,14 @@ "wasm-pack": "0.13.1" } }, + "node_modules/@action-validator/cli": { + "resolved": "packages/cli", + "link": true + }, + "node_modules/@action-validator/core": { + "resolved": "packages/core", + "link": true + }, "node_modules/axios": { "version": "0.26.1", "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz", @@ -351,9 +362,37 @@ "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", "dev": true, "license": "ISC" + }, + "packages/cli": { + "name": "@action-validator/cli", + "version": "0.0.0-git", + "license": "GPL-3.0-only", + "bin": { + "action-validator": "cli.mjs" + }, + "devDependencies": { + "@action-validator/core": "file:../core" + }, + "peerDependencies": { + "@action-validator/core": "0.0.0-git" + } + }, + "packages/core": { + "name": "@action-validator/core", + "version": "0.0.0-git", + "license": "GPL-3.0-only" } }, "dependencies": { + "@action-validator/cli": { + "version": "file:packages/cli", + "requires": { + "@action-validator/core": "file:../core" + } + }, + "@action-validator/core": { + "version": "file:packages/core" + }, "axios": { "version": "0.26.1", "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz", diff --git a/package.json b/package.json index 8f8a0d4..45a579f 100644 --- a/package.json +++ b/package.json @@ -3,10 +3,12 @@ "private": true, "version": "0.0.0-git", "license": "GPL-3.0-only", + "workspaces": ["packages/*"], "scripts": { "build": "./scripts/build-wasm.sh", - "build:dev": "./scripts/build-wasm.sh", - "test": "node test/run.mjs", + "build:dev": "WASM_PACK_BUILD_FLAGS='--no-opt' ./scripts/build-wasm.sh", + "test": "./scripts/test-wasm.sh", + "test:dev": "WASM_PACK_BUILD_FLAGS='--no-opt' ./scripts/test-wasm.sh", "lint": "prettier --check .", "format": "prettier --write ." }, diff --git a/scripts/build-wasm.sh b/scripts/build-wasm.sh index 34695e2..c4e231d 100755 --- a/scripts/build-wasm.sh +++ b/scripts/build-wasm.sh @@ -1,8 +1,10 @@ #!/usr/bin/env bash +WASM_PACK_BUILD_FLAGS=$WASM_PACK_BUILD_FLAGS + set -euo pipefail -npx wasm-pack build --out-dir target/wasm-pack/build --no-typescript --target nodejs --features js +npx wasm-pack build $WASM_PACK_BUILD_FLAGS --out-dir target/wasm-pack/build --no-typescript --target nodejs --no-default-features --features js rm -rf packages/core/snippets cp -R target/wasm-pack/build/snippets packages/core/snippets cp target/wasm-pack/build/action_validator_bg.wasm packages/core/ diff --git a/scripts/test-wasm.sh b/scripts/test-wasm.sh new file mode 100644 index 0000000..8a5360a --- /dev/null +++ b/scripts/test-wasm.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -euo pipefail + +npm run build +cargo test -F test-js diff --git a/tests/snapshot_tests.rs b/tests/snapshot_tests.rs index 6af0e6a..9016a92 100644 --- a/tests/snapshot_tests.rs +++ b/tests/snapshot_tests.rs @@ -25,7 +25,7 @@ impl SnapshotTest { } } - #[cfg(not(feature = "save-snapshots"))] + #[cfg(not(feature = "test-save-snapshots"))] fn execute(self) { let stderr = fs::read_to_string(self.test_dir.join("stderr")).unwrap_or(String::from("")); @@ -40,6 +40,7 @@ impl SnapshotTest { }) .unwrap_or(0); + #[cfg(not(feature = "test-js"))] Command::cargo_bin(env!("CARGO_PKG_NAME")) .expect("binary to execute") .args(self.workflow_files) @@ -47,9 +48,18 @@ impl SnapshotTest { .stdout(stdout) .stderr(stderr) .code(exitcode); + + #[cfg(feature = "test-js")] + Command::new("npx") + .arg("action-validator") + .args(self.workflow_files) + .assert() + .stdout(stdout) + .stderr(stderr) + .code(exitcode); } - #[cfg(feature = "save-snapshots")] + #[cfg(feature = "test-save-snapshots")] fn execute(&self) { use std::fs::File; use std::io::prelude::*; From d524e6dfb30977ec0e2ee078744445f00dd9d0b4 Mon Sep 17 00:00:00 2001 From: Ben Heidemann Date: Sun, 2 Mar 2025 15:05:12 +0000 Subject: [PATCH 06/20] docs: update contributing guide --- CONTRIBUTING.md | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1593235..e67a2be 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,12 +16,12 @@ # Environment Setup ## Install Rust -Firstly, you'll need make any changes to the core functionality of this project. We recommend use `rustup`, on the recommendation of the rust team. You can find the installation instructions [here](https://www.rust-lang.org/tools/install). +Firstly, you'll need a Rust toolchain to make any changes to the core functionality of this project. We recommend [using `rustup`](https://www.rust-lang.org/tools/install), because that's what the Rust core team recommend. To confirm that rust is installed, run the `cargo` command. If you don't receive the help docs output, you may need to add rust to your shell rc file. ## Git Submodule Setup -This repository uses [git submodules](https://git-scm.com/book/en/v2/Git-Tools-Submodules). Specifically for the use of [schemastore](https://github.com/SchemaStore/schemastore). +This repository uses [git submodules](https://git-scm.com/book/en/v2/Git-Tools-Submodules). Specifically for the use of [schemastore](https://github.com/SchemaStore/schemastore). To setup the git submodule after cloning this repo to your local, you'll want to run the following commands: 1. `git submodule init` @@ -38,7 +38,14 @@ Cloning into '/Users/someuser/action-validator/src/schemastore'... Submodule path 'src/schemastore': checked out 'd3e6ab7727380b214acbab05570fb09a3e5d2dfc' ``` -At this point, you should be all set to `cargo run`! If you run into any issues here, please [create an issue](https://github.com/mpalmer/action-validator/issues/new/choose). +At this point, you should be all set to `cargo run`! + +## Node/WASM Setup +If you plan to work on the WASM/Node bindings, you'll also need to install Node. We recommend using a [NVM](https://github.com/nvm-sh/nvm) to install the Node version listed in `.nvmrc`. + +Once Node is installed, run `npm install` at the root of the repository. + +You should now be all set to run `npm build`, to build the Node/WASM bindings. Once built, run `npx action-validator` to run the CLI via the Node/WASM bindings. # Running the Validator Locally @@ -71,6 +78,7 @@ Options: ``` # Writing Tests + All tests live in the `tests` directory. Currently, this project implements snapshot testing, but that's not to say you couldn't write unit or integration tests with the current structure. To run the tests, simply run `cargo test` from the root directory. If you want to test a specific @@ -86,8 +94,14 @@ When the tests is run, the results of the test must exactly match those of the p the snapshot tests are named in the format `{next_id}_{whats_being_tested}` (e.g. `011_remote_checks_failure`). If you have made changes which will change the output of the program and cause snapshots to fail, you can run -`cargo test -F save-snapshots`. This feature causes the executed command to save the `stdout`, `stderr`, and/or +`cargo test -F test-save-snapshots`. This feature causes the executed command to save the `stdout`, `stderr`, and/or exit code to the specified testing directory. If you are writing a net new test, you will need to create the test directory with your workflow or action file. -Once you're done, you can save the results to that directy by running `cargo test -F save-snapshots`. +Once you're done, you can save the results to that directy by running `cargo test -F test-save-snapshots`. + +# Testing Node/WASM Bindings + +To test against the Node/WASM bindings, you can run `npm test`, or `npm test:dev` (to skip optimisations). +Note that Node support is considered experiemental, and does not have one to one feature parity with the native binary yet. +As such, some tests may fail, even on `main`. From 8324a562f3d11a352ebbdd98a3fbbea5a62df149 Mon Sep 17 00:00:00 2001 From: Ben Heidemann Date: Sun, 2 Mar 2025 15:14:41 +0000 Subject: [PATCH 07/20] ci: run tests with cargo --- .github/workflows/audit.yml | 43 +++++++++++++ .github/workflows/qa.yml | 10 +-- test/run | 56 ---------------- test/run.mjs | 125 ------------------------------------ 4 files changed, 46 insertions(+), 188 deletions(-) delete mode 100755 test/run delete mode 100644 test/run.mjs diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index 3f7ba19..a47d1c3 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -70,3 +70,46 @@ jobs: run: | cargo +${{ steps.rust-install.outputs.name }} clippy -- -D warnings cargo +${{ steps.rust-install.outputs.name }} clippy --all-features -- -D warnings + + nightly_compat_node: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: true + + - name: Install rust + id: rust-install + uses: dtolnay/rust-toolchain@nightly + + - name: Update deps + run: | + cargo +${{ steps.rust-install.outputs.name }} update + + - name: Install rust + id: rust-install + uses: dtolnay/rust-toolchain@stable + + - name: Setup Node + uses: actions/setup-node@v4 + with: + cache: npm + node-version: nightly + + - name: Update Dependencies + run: npm update + + - name: Install Dependencies + run: npm install + + - name: Build + run: npm run build + # nodejs support is experimental; don't fail the build for it + continue-on-error: true + + - name: Test + run: npm test + # nodejs support is experimental; don't fail the build for it + continue-on-error: true diff --git a/.github/workflows/qa.yml b/.github/workflows/qa.yml index ea820cc..dae4ea3 100644 --- a/.github/workflows/qa.yml +++ b/.github/workflows/qa.yml @@ -56,7 +56,7 @@ jobs: uses: mfinelli/setup-shfmt@master - name: Run shfmt - run: shfmt -d bin/* test/run + run: shfmt -d bin/* - name: Setup Node uses: actions/setup-node@v4 @@ -140,13 +140,9 @@ jobs: with: toolchain: ${{ matrix.rust-toolchain }} - - name: Build - run: | - cargo +${{ steps.rust-install.outputs.name }} build - - name: Run Testsuite - working-directory: "test" - run: ./run + run: | + cargo +${{ steps.rust-install.outputs.name }} test build-test-node: runs-on: ubuntu-latest diff --git a/test/run b/test/run deleted file mode 100755 index f249e41..0000000 --- a/test/run +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -cd "$(dirname "${BASH_SOURCE[0]}")" - -readonly cmd="../target/debug/action-validator" - -if ! [ -x "$cmd" ]; then - echo "No action-validator to test; have you run 'cargo build' lately?" >&2 - exit 1 -fi - -for t in [0-9]*; do - expected_stdout="$(cat "$t/stdout" 2>/dev/null || true)" - expected_stderr="$(cat "$t/stderr" 2>/dev/null || true)" - expected_exitcode="$(cat "$t/exitcode" 2>/dev/null || echo 0)" - - dir="$(mktemp -d)" - trap 'rm -rf "$dir"' EXIT - - failed=n - rv=0 - - readarray -t -d '' testfiles < <(find "$t"/ -name '*.yml' -print0) - - "$cmd" "${testfiles[@]}" >"$dir/stdout" 2>"$dir/stderr" || rv="$?" - stdout="$(cat "$dir"/stdout)" - stderr="$(cat "$dir"/stderr)" - - if [ "$rv" != "$expected_exitcode" ]; then - echo "Test $t exited with unexpected code: expected $expected_exitcode, got $rv" >&2 - failed=y - fi - - if [ "$stdout" != "$expected_stdout" ]; then - echo "Test $t produced unexpected stdout:" >&2 - diff -u <(echo "$expected_stdout") <(echo "$stdout") >&2 || true - failed=y - fi - - if [ "$stderr" != "$expected_stderr" ]; then - echo "Test $t produced unexpected stderr:" >&2 - diff -u <(echo "$expected_stderr") <(echo "$stderr") >&2 || true - failed=y - fi - - rm -rf "$dir" - trap "" EXIT - - if [ "$failed" = "y" ]; then - exit 1 - fi -done - -echo "All tests passed." diff --git a/test/run.mjs b/test/run.mjs deleted file mode 100644 index 7b026e6..0000000 --- a/test/run.mjs +++ /dev/null @@ -1,125 +0,0 @@ -// @ts-check -import chalk from "chalk"; -import * as diff from "diff"; -import fs from "fs"; -import * as action_validator from "../packages/core/action_validator.js"; - -const update = process.argv.includes("--update") || process.argv.includes("-u"); - -const start = Date.now(); - -const passed = fs - .readdirSync("test") - // get all directories in test/ - .map((entry) => `test/${entry}`) - .filter((entry) => fs.statSync(entry).isDirectory()) - // get the first .yml files in each directory - .map((testDir) => [testDir, ...fs.readdirSync(testDir)]) - .map(([testDir, ...entries]) => [ - testDir, - entries.filter((entry) => entry.endsWith(".yml"))[0], - ]) - // get the test fixture and snapshot files for each directory - .map(([testDir, testFixtureFileName]) => [ - `${testDir}/${testFixtureFileName}`, - `${testDir}/validation_state.snap.json`, - ]) - // read the files - .map(([testFixtureFile, testSnapshotFile]) => [ - testFixtureFile, - fs.readFileSync(testFixtureFile, "utf8"), - testSnapshotFile, - fs.readFileSync(testSnapshotFile, "utf8"), - ]) - // validate the test fixture files - .map(([testFixtureFile, testFixture, testSnapshotFile, oldSnapshot]) => { - let validationState; - - // treat action.yml files as actions and everything else as workflows - if (testFixtureFile.endsWith("action.yml")) { - console.log( - chalk.bold(` > ${testFixtureFile} `) + - chalk.bold.gray("(action file)") + - "\n" - ); - console.log(chalk.bold.gray("=== OUTPUT START ===")); - validationState = action_validator.validateAction(testFixture); - console.log(chalk.bold.gray("=== OUTPUT END ===")); - } else { - console.log( - chalk.bold(` > ${testFixtureFile} `) + - chalk.bold.gray("(workflow file)") + - "\n" - ); - console.log(chalk.bold.gray("=== OUTPUT START ===")); - validationState = action_validator.validateWorkflow(testFixture); - console.log(chalk.bold.gray("=== OUTPUT END ===")); - } - - // stringify the validation state and compare it to the snapshot - const newSnapshot = JSON.stringify(validationState, null, 2) + "\n"; - const isSnapshotMatch = newSnapshot === oldSnapshot; - - if (isSnapshotMatch) { - console.log(chalk.bold.green("\n SNAPSHOT MATCHED")); - } else { - // print the diff if the snapshot doesn't match - console.log( - chalk.bold[update ? "yellow" : "red"]("\n SNAPSHOT MISMATCH\n") - ); - console.log( - `${chalk.bold.gray("Diff:")} ${chalk.bold.red("red")} ${chalk.gray( - "for deletions," - )} ${chalk.bold.green("green")} ${chalk.gray( - "for additions," - )} ${chalk.bold.gray("grey")} ${chalk.gray("for common parts")}` - ); - diff.diffJson(oldSnapshot, newSnapshot).forEach((part) => { - const color = part.added ? "green" : part.removed ? "red" : "grey"; - process.stderr.write(chalk[color](part.value)); - }); - process.stderr.write("\n"); - } - - console.log("\n"); - - return [isSnapshotMatch, testFixtureFile, testSnapshotFile, newSnapshot]; - }) - .map( - update - ? // update any snapshots that don't match (or create new ones) - ([isSnapshotMatch, testFixtureFile, testSnapshotFile, newSnapshot]) => { - if (!isSnapshotMatch) { - console.log( - chalk.bold.yellow(`Updating snapshot for ${testFixtureFile}`) - ); - - // @ts-ignore -- no const assertions in JS files =( - fs.writeFileSync(testSnapshotFile, newSnapshot); - } - - return true; - } - : // check if any snapshots don't match - ([isSnapshotMatch, testFixtureFile]) => { - if (!isSnapshotMatch) { - console.log( - chalk.bold.red(`Snapshot mismatch for ${testFixtureFile}`) - ); - } - - return update || isSnapshotMatch; - } - ) - // check if all tests passed - .every((passed) => passed); - -// exit with a non-zero exit code if any tests failed -if (!passed) { - console.log(chalk.bold.red("\nFailed some tests")); - - process.exit(1); -} - -const end = Date.now(); -console.log(chalk.bold.green(`\nPassed all tests in ${(end - start) / 1000}s`)); From 70192f37e63d9a370c01ef94cf1da49e8b694067 Mon Sep 17 00:00:00 2001 From: Ben Heidemann Date: Sat, 6 Sep 2025 19:39:35 +0100 Subject: [PATCH 08/20] ci: fix tests & skip by target --- packages/cli/cli.mjs | 0 scripts/build-wasm.sh | 6 +-- scripts/test-wasm.sh | 3 +- tests/fixtures/003_successful_globs/test.json | 4 +- tests/fixtures/004_failing_globs/test.json | 4 +- .../004a_failing_negative_glob/test.json | 4 +- tests/fixtures/011_subdir_globs/test.json | 1 + tests/snapshot_tests.rs | 42 ++++++++++++++++++- 8 files changed, 55 insertions(+), 9 deletions(-) mode change 100644 => 100755 packages/cli/cli.mjs mode change 100644 => 100755 scripts/test-wasm.sh diff --git a/packages/cli/cli.mjs b/packages/cli/cli.mjs old mode 100644 new mode 100755 diff --git a/scripts/build-wasm.sh b/scripts/build-wasm.sh index c4e231d..a3135e5 100755 --- a/scripts/build-wasm.sh +++ b/scripts/build-wasm.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash -WASM_PACK_BUILD_FLAGS=$WASM_PACK_BUILD_FLAGS - set -euo pipefail -npx wasm-pack build $WASM_PACK_BUILD_FLAGS --out-dir target/wasm-pack/build --no-typescript --target nodejs --no-default-features --features js +# [SC2086] Intenitonally omitting double quotes to allow word splitting of WASM_PACK_BUILD_FLAGS +# shellcheck disable=SC2086 +npx wasm-pack build ${WASM_PACK_BUILD_FLAGS:-} --out-dir target/wasm-pack/build --no-typescript --target nodejs --no-default-features --features js rm -rf packages/core/snippets cp -R target/wasm-pack/build/snippets packages/core/snippets cp target/wasm-pack/build/action_validator_bg.wasm packages/core/ diff --git a/scripts/test-wasm.sh b/scripts/test-wasm.sh old mode 100644 new mode 100755 index 8a5360a..d1762a5 --- a/scripts/test-wasm.sh +++ b/scripts/test-wasm.sh @@ -2,5 +2,4 @@ set -euo pipefail -npm run build -cargo test -F test-js +cargo test -F test-js -- --nocapture diff --git a/tests/fixtures/003_successful_globs/test.json b/tests/fixtures/003_successful_globs/test.json index 0967ef4..326816c 100644 --- a/tests/fixtures/003_successful_globs/test.json +++ b/tests/fixtures/003_successful_globs/test.json @@ -1 +1,3 @@ -{} +{ + "targets": { "node": false } +} diff --git a/tests/fixtures/004_failing_globs/test.json b/tests/fixtures/004_failing_globs/test.json index 0967ef4..326816c 100644 --- a/tests/fixtures/004_failing_globs/test.json +++ b/tests/fixtures/004_failing_globs/test.json @@ -1 +1,3 @@ -{} +{ + "targets": { "node": false } +} diff --git a/tests/fixtures/004a_failing_negative_glob/test.json b/tests/fixtures/004a_failing_negative_glob/test.json index 0967ef4..326816c 100644 --- a/tests/fixtures/004a_failing_negative_glob/test.json +++ b/tests/fixtures/004a_failing_negative_glob/test.json @@ -1 +1,3 @@ -{} +{ + "targets": { "node": false } +} diff --git a/tests/fixtures/011_subdir_globs/test.json b/tests/fixtures/011_subdir_globs/test.json index 60b9ffd..7bb141c 100644 --- a/tests/fixtures/011_subdir_globs/test.json +++ b/tests/fixtures/011_subdir_globs/test.json @@ -1,3 +1,4 @@ { + "targets": { "node": false }, "cli_args": ["--rootdir", "tests/fixtures/011_subdir_globs/subdir", "tests/fixtures/011_subdir_globs/subdir/glob.yml"] } diff --git a/tests/snapshot_tests.rs b/tests/snapshot_tests.rs index 4fce82a..49712cf 100644 --- a/tests/snapshot_tests.rs +++ b/tests/snapshot_tests.rs @@ -9,11 +9,36 @@ use fixtures::fixtures; static REPO_DIR_WILDCARD: &str = "{{repo}}"; +#[derive(Debug, serde::Deserialize)] +struct Targets { + #[cfg(not(feature = "test-js"))] + #[serde(default = "default_target_enabled")] + native: bool, + #[cfg(feature = "test-js")] + #[serde(default = "default_target_enabled")] + node: bool, +} + +fn default_target_enabled() -> bool { + true +} + #[derive(Debug, serde::Deserialize)] struct SnapshotTestConfig { + #[serde(default = "default_targets_config")] + targets: Targets, cli_args: Option>, } +fn default_targets_config() -> Targets { + Targets { + #[cfg(not(feature = "test-js"))] + native: true, + #[cfg(feature = "test-js")] + node: true, + } +} + #[derive(Debug)] struct SnapshotTest { config: SnapshotTestConfig, @@ -51,13 +76,28 @@ impl SnapshotTest { #[cfg(feature = "test-js")] { - Command::new("npx").arg("action-validator") + let mut cmd = Command::new("npx"); + cmd.arg("@action-validator/cli"); + cmd } } fn execute(self) { use std::ffi::OsString; + #[cfg(not(feature = "test-js"))] + let skip = self.config.targets.native == false; + #[cfg(feature = "test-js")] + let skip = self.config.targets.node == false; + + if skip { + eprintln!( + "test snapshot_{} ... skipped", + self.test_dir.file_name().unwrap().to_str().unwrap() + ); + return; + } + let pwd = self.current_dir.to_str().unwrap(); let stderr = fs::read_to_string(self.test_dir.join("stderr")) From 20decaa2cf097807131ee48116029f76f3bff558 Mon Sep 17 00:00:00 2001 From: Ben Heidemann Date: Sat, 6 Sep 2025 19:43:23 +0100 Subject: [PATCH 09/20] ci: log skipped tests when running native test suite --- .github/workflows/audit.yml | 4 ++-- .github/workflows/qa.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index a47d1c3..ba48566 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -63,8 +63,8 @@ jobs: env: RUSTFLAGS: -D warnings run: | - cargo +${{ steps.rust-install.outputs.name }} test - cargo +${{ steps.rust-install.outputs.name }} test --all-features + cargo +${{ steps.rust-install.outputs.name }} test -- --nocapture + cargo +${{ steps.rust-install.outputs.name }} test --all-features -- --nocapture - name: Clippy run: | diff --git a/.github/workflows/qa.yml b/.github/workflows/qa.yml index d479f11..1934cb6 100644 --- a/.github/workflows/qa.yml +++ b/.github/workflows/qa.yml @@ -144,7 +144,7 @@ jobs: - name: Run Testsuite run: | - cargo +${{ steps.rust-install.outputs.name }} test + cargo +${{ steps.rust-install.outputs.name }} test -- --nocapture build-test-node: runs-on: ubuntu-latest From e69a0895a1dc10dc0a24d8ada66181e3a1eb2551 Mon Sep 17 00:00:00 2001 From: Ben Heidemann Date: Sat, 6 Sep 2025 19:46:30 +0100 Subject: [PATCH 10/20] test: fix test-save-snapshots feature --- tests/snapshot_tests.rs | 51 ++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/tests/snapshot_tests.rs b/tests/snapshot_tests.rs index 49712cf..1fc5ec1 100644 --- a/tests/snapshot_tests.rs +++ b/tests/snapshot_tests.rs @@ -100,23 +100,6 @@ impl SnapshotTest { let pwd = self.current_dir.to_str().unwrap(); - let stderr = fs::read_to_string(self.test_dir.join("stderr")) - .unwrap_or(String::from("")) - .replace(REPO_DIR_WILDCARD, pwd); - - let stdout = fs::read_to_string(self.test_dir.join("stdout")) - .unwrap_or(String::from("")) - .replace(REPO_DIR_WILDCARD, pwd); - - let exitcode: i32 = fs::read_to_string(self.test_dir.join("exitcode")) - .map(|s| { - s.strip_suffix("\n") - .unwrap_or(s.as_str()) - .parse::() - .unwrap_or(0) - }) - .unwrap_or(0); - let cli_args: Vec<_> = if let Some(cli_args) = &self.config.cli_args { cli_args.iter().map(OsString::from).collect() } else { @@ -129,12 +112,31 @@ impl SnapshotTest { }; #[cfg(not(feature = "test-save-snapshots"))] - self.build_command() - .args(&cli_args) - .assert() - .stdout(stdout) - .stderr(stderr) - .code(exitcode); + { + let stderr = fs::read_to_string(self.test_dir.join("stderr")) + .unwrap_or(String::from("")) + .replace(REPO_DIR_WILDCARD, pwd); + + let stdout = fs::read_to_string(self.test_dir.join("stdout")) + .unwrap_or(String::from("")) + .replace(REPO_DIR_WILDCARD, pwd); + + let exitcode: i32 = fs::read_to_string(self.test_dir.join("exitcode")) + .map(|s| { + s.strip_suffix("\n") + .unwrap_or(s.as_str()) + .parse::() + .unwrap_or(0) + }) + .unwrap_or(0); + + self.build_command() + .args(&cli_args) + .assert() + .stdout(stdout) + .stderr(stderr) + .code(exitcode); + } #[cfg(feature = "test-save-snapshots")] { @@ -178,9 +180,6 @@ impl SnapshotTest { } } } - - #[cfg(feature = "test-save-snapshots")] - fn execute(&self) {} } #[fixtures(["tests/fixtures/*"])] From 9b51807d206a3cda591c3f61a88cefb38b02dd8e Mon Sep 17 00:00:00 2001 From: Ben Heidemann Date: Sat, 6 Sep 2025 19:55:01 +0100 Subject: [PATCH 11/20] docs: update contributing docs for test.json config --- CONTRIBUTING.md | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e67a2be..b1e4eae 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -97,8 +97,26 @@ If you have made changes which will change the output of the program and cause s `cargo test -F test-save-snapshots`. This feature causes the executed command to save the `stdout`, `stderr`, and/or exit code to the specified testing directory. -If you are writing a net new test, you will need to create the test directory with your workflow or action file. -Once you're done, you can save the results to that directy by running `cargo test -F test-save-snapshots`. +If you are writing a net new test, you will need to create the test directory with your workflow or action file, and a +`test.json` file. Once you're done, you can save the results to that directy by running +`cargo test -F test-save-snapshots`. + +The `test.json` file contains the test configuration. It can usually be left empty (i.e. `{}`). + +```jsonc +// Example test.json +{ + "targets": { + "node": false, + "native": true + }, + "cli_args": [ + "--rootdir", + "tests/fixtures/011_subdir_globs/subdir", + "tests/fixtures/011_subdir_globs/subdir/glob.yml" + ] +} +``` # Testing Node/WASM Bindings From 06db305ba91e84e2c755524117643678ab85743d Mon Sep 17 00:00:00 2001 From: Ben Heidemann Date: Fri, 3 Oct 2025 17:28:23 +0100 Subject: [PATCH 12/20] chore: update to latest fixtures --- Cargo.lock | 16 ++++++++++++---- Cargo.toml | 4 ++-- build.rs | 4 +--- tests/snapshot_tests.rs | 1 + 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3b465ff..61ba8d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -349,23 +349,25 @@ checksum = "7fd99930f64d146689264c637b5af2f0233a933bef0d8570e2526bf9e083192d" [[package]] name = "fixtures" -version = "1.0.2" +version = "2.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48ef9d43f47935c5fa539d5768fd95f41c95230e60751183d6370466a288a49e" +checksum = "00416a7dbf97a06d8371ccb4bed09ec1bd2ff152d93dc370b7c93a85f58ec8aa" dependencies = [ "fixtures_proc", ] [[package]] name = "fixtures_proc" -version = "1.0.2" +version = "2.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a91a11e9f25364088337fa5f7224f03d7a194b4ed18cfed326bd4d82808ebd50" +checksum = "9c33e602c3f9a64e16fca051e511ddfdd29b8f9e988843ee38d3acc0cada29a7" dependencies = [ + "globset", "globwalk", "proc-macro2", "quote", "syn", + "unicode-xid", ] [[package]] @@ -1015,6 +1017,12 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "unsafe-libyaml" version = "0.2.11" diff --git a/Cargo.toml b/Cargo.toml index 40d2b97..9a5339a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,10 +49,10 @@ assert_cmd = { version = "2.0.8", optional = true } [dev-dependencies] wasm-bindgen-test = "0.3.34" -fixtures = "1.0.2" +fixtures = "2.3.5" [build-dependencies] -fixtures = "1.0.2" +fixtures = "2.3.5" [profile.release] # Tell `rustc` to optimize for small code size. diff --git a/build.rs b/build.rs index 36a65e8..97e75b6 100644 --- a/build.rs +++ b/build.rs @@ -1,5 +1,3 @@ -use fixtures::build::watch_fixture_dir; - fn main() { - watch_fixture_dir("tests/fixtures"); + fixtures::build::watch_dir("tests/fixtures"); } diff --git a/tests/snapshot_tests.rs b/tests/snapshot_tests.rs index 1fc5ec1..a810994 100644 --- a/tests/snapshot_tests.rs +++ b/tests/snapshot_tests.rs @@ -183,6 +183,7 @@ impl SnapshotTest { } #[fixtures(["tests/fixtures/*"])] +#[test] fn snapshot(dir: &Path) { SnapshotTest::new(dir).execute(); } From 81133cd0ce53ed2fcda1e9c20e30893ec500e2b6 Mon Sep 17 00:00:00 2001 From: Ben Heidemann Date: Sat, 4 Oct 2025 14:33:41 +0100 Subject: [PATCH 13/20] Empty commit From 0c157dfc333fd8edc636ab84e95f22adc6e2b968 Mon Sep 17 00:00:00 2001 From: Ben Heidemann Date: Sun, 5 Oct 2025 10:03:17 +0100 Subject: [PATCH 14/20] refactor: remove build dependency on fixtures --- Cargo.toml | 3 --- build.rs | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9a5339a..4e4bddc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,9 +51,6 @@ assert_cmd = { version = "2.0.8", optional = true } wasm-bindgen-test = "0.3.34" fixtures = "2.3.5" -[build-dependencies] -fixtures = "2.3.5" - [profile.release] # Tell `rustc` to optimize for small code size. opt-level = "s" diff --git a/build.rs b/build.rs index 97e75b6..e998592 100644 --- a/build.rs +++ b/build.rs @@ -1,3 +1,3 @@ fn main() { - fixtures::build::watch_dir("tests/fixtures"); + println!("cargo:rerun-if-changed=tests/fixtures"); } From 0e4ed5ebf7fdfc73a7eecab42cfa44956f92136c Mon Sep 17 00:00:00 2001 From: Ben Heidemann Date: Sun, 5 Oct 2025 10:21:40 +0100 Subject: [PATCH 15/20] test: remove custom implementation for skipping by feature The following can now be used (as of fixtures 2.4.0): ```rs #[cfg_attr( feature = "js", fixtures::ignore( paths = "tests/fixtures/999_unsuported_feature", reason = "not supported yet" ) )] ``` --- Cargo.lock | 75 ++++++++++++++----- Cargo.toml | 2 +- tests/fixtures/003_successful_globs/test.json | 4 +- tests/fixtures/004_failing_globs/test.json | 4 +- .../004a_failing_negative_glob/test.json | 4 +- tests/fixtures/011_subdir_globs/test.json | 1 - tests/snapshot_tests.rs | 38 ---------- 7 files changed, 62 insertions(+), 66 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 61ba8d7..0c4df35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -231,7 +231,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.106", ] [[package]] @@ -301,7 +301,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.106", ] [[package]] @@ -349,24 +349,25 @@ checksum = "7fd99930f64d146689264c637b5af2f0233a933bef0d8570e2526bf9e083192d" [[package]] name = "fixtures" -version = "2.3.5" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00416a7dbf97a06d8371ccb4bed09ec1bd2ff152d93dc370b7c93a85f58ec8aa" +checksum = "c3d1253329670ef6ee65e8d72161a1fae66bc8ea23c12d40522cbdf5c2c35064" dependencies = [ "fixtures_proc", ] [[package]] name = "fixtures_proc" -version = "2.3.5" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c33e602c3f9a64e16fca051e511ddfdd29b8f9e988843ee38d3acc0cada29a7" +checksum = "d7d75770cda1757c6e2f63dc4fd1b8181836fae2d850793ae79d8dbafc5f2b89" dependencies = [ "globset", "globwalk", + "proc-macro-error", "proc-macro2", "quote", - "syn", + "syn 2.0.106", "unicode-xid", ] @@ -783,6 +784,30 @@ dependencies = [ "termtree", ] +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro2" version = "1.0.101" @@ -915,7 +940,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.106", ] [[package]] @@ -973,6 +998,16 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.106" @@ -992,7 +1027,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.106", ] [[package]] @@ -1097,6 +1132,12 @@ dependencies = [ "uuid", ] +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "wait-timeout" version = "0.2.1" @@ -1148,7 +1189,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn", + "syn 2.0.106", "wasm-bindgen-shared", ] @@ -1183,7 +1224,7 @@ checksum = "7bb4ce89b08211f923caf51d527662b75bdc9c9c7aab40f86dcb9fb85ac552aa" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.106", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -1218,7 +1259,7 @@ checksum = "c5ada2ab788d46d4bda04c9d567702a79c8ced14f51f221646a16ed39d0e6a5d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.106", ] [[package]] @@ -1261,7 +1302,7 @@ checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.106", ] [[package]] @@ -1272,7 +1313,7 @@ checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.106", ] [[package]] @@ -1478,7 +1519,7 @@ checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.106", "synstructure", ] @@ -1499,7 +1540,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.106", "synstructure", ] @@ -1533,5 +1574,5 @@ checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.106", ] diff --git a/Cargo.toml b/Cargo.toml index 4e4bddc..054d8c8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,7 +49,7 @@ assert_cmd = { version = "2.0.8", optional = true } [dev-dependencies] wasm-bindgen-test = "0.3.34" -fixtures = "2.3.5" +fixtures = "2.4.0" [profile.release] # Tell `rustc` to optimize for small code size. diff --git a/tests/fixtures/003_successful_globs/test.json b/tests/fixtures/003_successful_globs/test.json index 326816c..0967ef4 100644 --- a/tests/fixtures/003_successful_globs/test.json +++ b/tests/fixtures/003_successful_globs/test.json @@ -1,3 +1 @@ -{ - "targets": { "node": false } -} +{} diff --git a/tests/fixtures/004_failing_globs/test.json b/tests/fixtures/004_failing_globs/test.json index 326816c..0967ef4 100644 --- a/tests/fixtures/004_failing_globs/test.json +++ b/tests/fixtures/004_failing_globs/test.json @@ -1,3 +1 @@ -{ - "targets": { "node": false } -} +{} diff --git a/tests/fixtures/004a_failing_negative_glob/test.json b/tests/fixtures/004a_failing_negative_glob/test.json index 326816c..0967ef4 100644 --- a/tests/fixtures/004a_failing_negative_glob/test.json +++ b/tests/fixtures/004a_failing_negative_glob/test.json @@ -1,3 +1 @@ -{ - "targets": { "node": false } -} +{} diff --git a/tests/fixtures/011_subdir_globs/test.json b/tests/fixtures/011_subdir_globs/test.json index 7bb141c..60b9ffd 100644 --- a/tests/fixtures/011_subdir_globs/test.json +++ b/tests/fixtures/011_subdir_globs/test.json @@ -1,4 +1,3 @@ { - "targets": { "node": false }, "cli_args": ["--rootdir", "tests/fixtures/011_subdir_globs/subdir", "tests/fixtures/011_subdir_globs/subdir/glob.yml"] } diff --git a/tests/snapshot_tests.rs b/tests/snapshot_tests.rs index a810994..1e2d459 100644 --- a/tests/snapshot_tests.rs +++ b/tests/snapshot_tests.rs @@ -9,36 +9,11 @@ use fixtures::fixtures; static REPO_DIR_WILDCARD: &str = "{{repo}}"; -#[derive(Debug, serde::Deserialize)] -struct Targets { - #[cfg(not(feature = "test-js"))] - #[serde(default = "default_target_enabled")] - native: bool, - #[cfg(feature = "test-js")] - #[serde(default = "default_target_enabled")] - node: bool, -} - -fn default_target_enabled() -> bool { - true -} - #[derive(Debug, serde::Deserialize)] struct SnapshotTestConfig { - #[serde(default = "default_targets_config")] - targets: Targets, cli_args: Option>, } -fn default_targets_config() -> Targets { - Targets { - #[cfg(not(feature = "test-js"))] - native: true, - #[cfg(feature = "test-js")] - node: true, - } -} - #[derive(Debug)] struct SnapshotTest { config: SnapshotTestConfig, @@ -85,19 +60,6 @@ impl SnapshotTest { fn execute(self) { use std::ffi::OsString; - #[cfg(not(feature = "test-js"))] - let skip = self.config.targets.native == false; - #[cfg(feature = "test-js")] - let skip = self.config.targets.node == false; - - if skip { - eprintln!( - "test snapshot_{} ... skipped", - self.test_dir.file_name().unwrap().to_str().unwrap() - ); - return; - } - let pwd = self.current_dir.to_str().unwrap(); let cli_args: Vec<_> = if let Some(cli_args) = &self.config.cli_args { From f1a9d25993a4eec8f0c644bd2d488abbe80ac81f Mon Sep 17 00:00:00 2001 From: Ben Heidemann Date: Sun, 5 Oct 2025 10:28:49 +0100 Subject: [PATCH 16/20] ci: remove --nocapture from test runs --- .github/workflows/audit.yml | 4 ++-- .github/workflows/qa.yml | 2 +- scripts/test-wasm.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index ba48566..a47d1c3 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -63,8 +63,8 @@ jobs: env: RUSTFLAGS: -D warnings run: | - cargo +${{ steps.rust-install.outputs.name }} test -- --nocapture - cargo +${{ steps.rust-install.outputs.name }} test --all-features -- --nocapture + cargo +${{ steps.rust-install.outputs.name }} test + cargo +${{ steps.rust-install.outputs.name }} test --all-features - name: Clippy run: | diff --git a/.github/workflows/qa.yml b/.github/workflows/qa.yml index 1934cb6..d479f11 100644 --- a/.github/workflows/qa.yml +++ b/.github/workflows/qa.yml @@ -144,7 +144,7 @@ jobs: - name: Run Testsuite run: | - cargo +${{ steps.rust-install.outputs.name }} test -- --nocapture + cargo +${{ steps.rust-install.outputs.name }} test build-test-node: runs-on: ubuntu-latest diff --git a/scripts/test-wasm.sh b/scripts/test-wasm.sh index d1762a5..8ff6e82 100755 --- a/scripts/test-wasm.sh +++ b/scripts/test-wasm.sh @@ -2,4 +2,4 @@ set -euo pipefail -cargo test -F test-js -- --nocapture +cargo test -F test-js From 76bb50689f516ea04b9356b70ba816ea5ce7d9a6 Mon Sep 17 00:00:00 2001 From: Ben Heidemann Date: Sun, 5 Oct 2025 10:29:16 +0100 Subject: [PATCH 17/20] test: ignore unsupported test cases for JS targets --- tests/snapshot_tests.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/snapshot_tests.rs b/tests/snapshot_tests.rs index 1e2d459..504d1e8 100644 --- a/tests/snapshot_tests.rs +++ b/tests/snapshot_tests.rs @@ -145,6 +145,25 @@ impl SnapshotTest { } #[fixtures(["tests/fixtures/*"])] +#[cfg_attr( + feature = "test-js", + fixtures::ignore( + paths = "tests/fixtures/003_successful_globs", + reason = "Glob support is not implemented for JS targets yet." + ), + fixtures::ignore( + paths = "tests/fixtures/004_failing_globs", + reason = "Glob support is not implemented for JS targets yet." + ), + fixtures::ignore( + paths = "tests/fixtures/004a_failing_negative_glob", + reason = "Glob support is not implemented for JS targets yet." + ), + fixtures::ignore( + paths = "tests/fixtures/011_subdir_globs", + reason = "Glob support is not implemented for JS targets yet." + ) +)] #[test] fn snapshot(dir: &Path) { SnapshotTest::new(dir).execute(); From d556cb86ea4ae26a8aeb7b105e2941908a8c4cd3 Mon Sep 17 00:00:00 2001 From: Ben Heidemann Date: Sun, 5 Oct 2025 11:04:45 +0100 Subject: [PATCH 18/20] Update CONTRIBUTING.md Co-authored-by: Matt Palmer --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b1e4eae..6369d96 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,7 +41,7 @@ Submodule path 'src/schemastore': checked out 'd3e6ab7727380b214acbab05570fb09a3 At this point, you should be all set to `cargo run`! ## Node/WASM Setup -If you plan to work on the WASM/Node bindings, you'll also need to install Node. We recommend using a [NVM](https://github.com/nvm-sh/nvm) to install the Node version listed in `.nvmrc`. +If you plan to work on the WASM/Node bindings, you'll also need to install Node. We recommend using [NVM](https://github.com/nvm-sh/nvm) to install the Node version listed in `.nvmrc`. Once Node is installed, run `npm install` at the root of the repository. From dffdd0d91c92ec2bd3ec825bcdd0b0d85eeae252 Mon Sep 17 00:00:00 2001 From: Ben Heidemann Date: Sun, 5 Oct 2025 11:04:52 +0100 Subject: [PATCH 19/20] Update CONTRIBUTING.md Co-authored-by: Matt Palmer --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6369d96..2fc8169 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -90,7 +90,7 @@ follow the directions in [this guide](https://doc.rust-lang.org/book/ch11-01-wri ## Snapshot Tests A snapshot test is performed when we execute the cli and capture `stdout`, `stderr`, and/or an exit code. -When the tests is run, the results of the test must exactly match those of the previous run. For this project, +When the tests are run, the results of the test must exactly match those of the previous run. For this project, the snapshot tests are named in the format `{next_id}_{whats_being_tested}` (e.g. `011_remote_checks_failure`). If you have made changes which will change the output of the program and cause snapshots to fail, you can run From 64575e4e5d18750cbf8737b9810f7956fdbf2bf2 Mon Sep 17 00:00:00 2001 From: Ben Heidemann Date: Sun, 5 Oct 2025 11:04:57 +0100 Subject: [PATCH 20/20] Update CONTRIBUTING.md Co-authored-by: Matt Palmer --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2fc8169..c9c0513 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -121,5 +121,5 @@ The `test.json` file contains the test configuration. It can usually be left emp # Testing Node/WASM Bindings To test against the Node/WASM bindings, you can run `npm test`, or `npm test:dev` (to skip optimisations). -Note that Node support is considered experiemental, and does not have one to one feature parity with the native binary yet. +Note that Node support is considered experimental, and does not have one to one feature parity with the native binary yet. As such, some tests may fail, even on `main`.