Merge #526

526: implement run-pass tests as xtasks r=korken89 a=Lotterleben

resolves https://github.com/rtic-rs/cortex-m-rtic/issues/499 .

With this PR, you should be able to run `cargo xtask --target <desired target>` or `cargo xtask --target all` locally. Of course, it also reconfigures the CI workflow to do the same.

Note that I've translated the old `Run-pass tests` verbatim for now, which means the code includes checks for a `"types"`example which doesn't exist anymore.

The examples could be collected much more nicely to prevent leftovers like this in the future, but imo that could also be achieved in a separate PR.

Co-authored-by: Lotte Steenbrink <lotte.steenbrink@ferrous-systems.com>

.cargo/config

@@ -1,3 +1,6 @@
+[alias]
+xtask = "run --package xtask --"
+
 [target.thumbv6m-none-eabi]
 runner = "qemu-system-arm -cpu cortex-m3 -machine lm3s6965evb -nographic -semihosting-config enable=on,target=native -kernel"
 

.github/workflows/build.yml

@@ -198,139 +198,10 @@ jobs:
           sudo apt update
           sudo apt install -y qemu-system-arm
 
+
       - name: Run-pass tests
-        run: |
-          # Print the path
-          echo $PATH
-
-          arm_example() {
-              local COMMAND=$1
-              local EXAMPLE=$2
-              local BUILD_MODE=$3
-              local FEATURES=$4
-              local BUILD_NUM=$5
-
-              if [ $BUILD_MODE = "release" ]; then
-                  local RELEASE_FLAG="--release"
-              else
-                  local RELEASE_FLAG=""
-              fi
-
-              if [ -n "$FEATURES" ]; then
-                  local FEATURES_FLAG="--features $FEATURES"
-                  local FEATURES_STR=${FEATURES/,/_}_
-              else
-                  local FEATURES_FLAG=""
-                  local FEATURES_STR=""
-              fi
-              local CARGO_FLAGS="--example $EXAMPLE --target ${{ matrix.target }} $RELEASE_FLAG $FEATURES_FLAG"
-
-              if [ $COMMAND = "run" ]; then
-                  cargo $COMMAND $CARGO_FLAGS | diff -u ci/expected/$EXAMPLE.run -
-              else
-                  cargo $COMMAND $CARGO_FLAGS
-              fi
-              cargo objcopy $CARGO_FLAGS  -- -O ihex ci/builds/${EXAMPLE}_${FEATURES_STR}${BUILD_MODE}_${BUILD_NUM}.hex
-          }
-
-          mkdir -p ci/builds
-          exs=(
-              idle
-              init
-              hardware
-              preempt
-              binds
-
-              resource
-              lock
-              multilock
-              only-shared-access
-
-              task
-              message
-              capacity
-
-              not-sync
-
-              generics
-              pool
-              ramfunc
-
-              peripherals-taken
-          )
-
-          for ex in ${exs[@]}; do
-              if [ $ex = pool ]; then
-                  if [ ${{ matrix.target }} = thumbv6m-none-eabi ]; then
-                      continue
-                  fi
-
-                  td=$(mktemp -d)
-
-                  cargo run --example $ex --target ${{ matrix.target }} --features __v7 >\
-                          $td/pool.run
-                  grep 'foo(0x2' $td/pool.run
-                  grep 'bar(0x2' $td/pool.run
-                  cargo objcopy --example $ex --target ${{ matrix.target }} --features __v7 -- -O ihex ci/builds/${ex}___v7_debug_1.hex
-
-                  cargo run --example $ex --target ${{ matrix.target }} --features __v7 --release >\
-                          $td/pool.run
-                  grep 'foo(0x2' $td/pool.run
-                  grep 'bar(0x2' $td/pool.run
-                  cargo objcopy --example $ex --target ${{ matrix.target }} --features __v7 --release -- -O ihex ci/builds/${ex}___v7_release_1.hex
-
-                  rm -rf $td
-
-                  continue
-              fi
-
-              if [ $ex = types ]; then
-                  if [ ${{ matrix.target }} = thumbv6m-none-eabi ]; then
-                      continue
-                  fi
-
-                  arm_example "run" $ex "debug" "__v7" "1"
-                  arm_example "run" $ex "release" "__v7" "1"
-
-                  continue
-              fi
-
-              arm_example "run" $ex "debug" "" "1"
-              if [ $ex = types ]; then
-                  arm_example "run" $ex "release" "" "1"
-              else
-                  arm_example "build" $ex "release" "" "1"
-              fi
-          done
-
-          built=()
-          cargo clean
-          for ex in ${exs[@]}; do
-              if [ $ex = types ] || [ $ex = pool ]; then
-                  if [ ${{ matrix.target }} = thumbv6m-none-eabi ]; then
-                      continue
-                  fi
-
-                  arm_example "build" $ex "debug" "__v7" "2"
-                  cmp ci/builds/${ex}___v7_debug_1.hex \
-                      ci/builds/${ex}___v7_debug_2.hex
-                  arm_example "build" $ex "release" "__v7" "2"
-                  cmp ci/builds/${ex}___v7_release_1.hex \
-                      ci/builds/${ex}___v7_release_2.hex
-              else
-                  arm_example "build" $ex "debug" "" "2"
-                  cmp ci/builds/${ex}_debug_1.hex \
-                      ci/builds/${ex}_debug_2.hex
-                  arm_example "build" $ex "release" "" "2"
-                  cmp ci/builds/${ex}_release_1.hex \
-                      ci/builds/${ex}_release_2.hex
-              fi
-
-              built+=( $ex )
-          done
-
-          ( cd target/${{ matrix.target }}/release/examples/ && size ${built[@]} )
-
+        run:
+          cargo xtask --target ${{ matrix.target }}
 
   # Check the correctness of macros/ crate
   checkmacros:

Cargo.toml

@@ -76,6 +76,7 @@ lto = true
 [workspace]
 members = [
   "macros",
+  "xtask",
 ]
 
 # do not optimize proc-macro deps or build scripts

README.md

@@ -86,6 +86,16 @@ New features and big changes should go through the RFC process in the
 
 [rfcs]: https://github.com/rtic-rs/rfcs
 
+## Running tests locally
+
+To check all `Run-pass tests` locally on your `thumbv6m-none-eabi` or `thumbv7m-none-eabi` target device, run
+
+```console
+$ cargo xtask --target <your target>
+#                       ˆˆˆˆˆˆˆˆˆˆˆˆ
+#                   e.g. thumbv7m-none-eabi
+```
+
 ## Acknowledgments
 
 This crate is based on the [Real-Time For the Masses language][rtfm-lang]

xtask/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "xtask"
+version = "0.1.0"
+edition = "2018"
+
+[dependencies]
+anyhow = "1.0.43"
+os_pipe = "0.9.2"
+structopt = "0.3.22"
+tempdir = "0.3.7"

xtask/src/build.rs

@@ -0,0 +1,53 @@
+use std::{
+    fs,
+    path::{Path, PathBuf},
+};
+
+use crate::{command::BuildMode, TestRunError};
+
+const HEX_BUILD_ROOT: &str = "ci/builds";
+
+/// make sure we're starting with a clean,but existing slate
+pub fn init_build_dir() -> anyhow::Result<()> {
+    if Path::new(HEX_BUILD_ROOT).exists() {
+        fs::remove_dir_all(HEX_BUILD_ROOT)
+            .map_err(|_| anyhow::anyhow!("Could not clear out directory:  {}", HEX_BUILD_ROOT))?;
+    }
+    fs::create_dir_all(HEX_BUILD_ROOT)
+        .map_err(|_| anyhow::anyhow!("Could not create directory:  {}", HEX_BUILD_ROOT))
+}
+
+pub fn build_hexpath(
+    example: &str,
+    features: Option<&str>,
+    build_mode: BuildMode,
+    build_num: u32,
+) -> anyhow::Result<String> {
+    let features = match features {
+        Some(f) => f,
+        None => "",
+    };
+
+    let filename = format!("{}_{}_{}_{}.hex", example, features, build_mode, build_num);
+
+    let mut path = PathBuf::from(HEX_BUILD_ROOT);
+    path.push(filename);
+
+    path.into_os_string()
+        .into_string()
+        .map_err(|e| anyhow::Error::new(TestRunError::PathConversionError(e)))
+}
+
+pub fn compare_builds(file_1: String, file_2: String) -> anyhow::Result<()> {
+    let buf_1 = std::fs::read_to_string(file_1.clone())?;
+    let buf_2 = std::fs::read_to_string(file_2.clone())?;
+
+    if buf_1 != buf_2 {
+        return Err(anyhow::Error::new(TestRunError::FileCmpError {
+            file_1,
+            file_2,
+        }));
+    }
+
+    Ok(())
+}

xtask/src/command.rs

@@ -0,0 +1,162 @@
+use crate::RunResult;
+use core::fmt;
+use os_pipe::pipe;
+use std::{fs::File, io::Read, path::Path, process::Command};
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum BuildMode {
+    Release,
+    Debug,
+}
+
+pub enum CargoCommand<'a> {
+    Run {
+        example: &'a str,
+        target: &'a str,
+        features: Option<&'a str>,
+        mode: BuildMode,
+    },
+    Build {
+        example: &'a str,
+        target: &'a str,
+        features: Option<&'a str>,
+        mode: BuildMode,
+    },
+    Objcopy {
+        example: &'a str,
+        target: &'a str,
+        features: Option<&'a str>,
+        ihex: &'a str,
+    },
+    Size {
+        example_paths: Vec<&'a Path>,
+    },
+    Clean,
+}
+
+impl<'a> CargoCommand<'a> {
+    fn name(&self) -> &str {
+        match self {
+            CargoCommand::Run { .. } => "run",
+            CargoCommand::Size { example_paths: _ } => "rust-size",
+            CargoCommand::Clean => "clean",
+            CargoCommand::Build { .. } => "build",
+            CargoCommand::Objcopy { .. } => "objcopy",
+        }
+    }
+
+    pub fn args(&self) -> Vec<&str> {
+        match self {
+            CargoCommand::Run {
+                example,
+                target,
+                features,
+                mode,
+            }
+            | CargoCommand::Build {
+                example,
+                target,
+                features,
+                mode,
+            } => {
+                let mut args = vec![self.name(), "--example", example, "--target", target];
+
+                if let Some(feature_name) = features {
+                    args.extend_from_slice(&["--features", feature_name]);
+                }
+                if let Some(flag) = mode.to_flag() {
+                    args.push(flag);
+                }
+                args
+            }
+            CargoCommand::Size { example_paths } => {
+                example_paths.iter().map(|p| p.to_str().unwrap()).collect()
+            }
+            CargoCommand::Clean => vec!["clean"],
+            CargoCommand::Objcopy {
+                example,
+                target,
+                features,
+                ihex,
+            } => {
+                let mut args = vec![self.name(), "--example", example, "--target", target];
+
+                if let Some(feature_name) = features {
+                    args.extend_from_slice(&["--features", feature_name]);
+                }
+
+                // this always needs to go at the end
+                args.extend_from_slice(&["--", "-O", "ihex", ihex]);
+                args
+            }
+        }
+    }
+
+    pub fn command(&self) -> &str {
+        match self {
+            // we need to cheat a little here:
+            // `cargo size` can't be ran on multiple files, so we're using `rust-size` instead –
+            // which isn't a command that starts wizh `cargo`. So we're sneakily swapping them out :)
+            CargoCommand::Size { .. } => "rust-size",
+            _ => "cargo",
+        }
+    }
+}
+
+impl BuildMode {
+    pub fn to_flag(&self) -> Option<&str> {
+        match self {
+            BuildMode::Release => Some("--release"),
+            BuildMode::Debug => None,
+        }
+    }
+}
+
+impl fmt::Display for BuildMode {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let cmd = match self {
+            BuildMode::Release => "release",
+            BuildMode::Debug => "debug",
+        };
+
+        write!(f, "{}", cmd)
+    }
+}
+
+pub fn run_command(command: &CargoCommand) -> anyhow::Result<RunResult> {
+    let (mut reader, writer) = pipe()?;
+    println!("👟 {} {}", command.command(), command.args().join(" "));
+
+    let mut handle = Command::new(command.command())
+        .args(command.args())
+        .stdout(writer)
+        .spawn()?;
+
+    // retrieve output and clean up
+    let mut output = String::new();
+    reader.read_to_string(&mut output)?;
+    let exit_status = handle.wait()?;
+
+    Ok(RunResult {
+        exit_status,
+        output,
+    })
+}
+
+/// Check if `run` was sucessful.
+/// returns Ok in case the run went as expected,
+/// Err otherwise
+pub fn run_successful(run: &RunResult, expected_output_file: String) -> anyhow::Result<()> {
+    let mut file_handle = File::open(expected_output_file)?;
+    let mut expected_output = String::new();
+    file_handle.read_to_string(&mut expected_output)?;
+    if expected_output == run.output && run.exit_status.success() {
+        Ok(())
+    } else {
+        Err(anyhow::anyhow!(
+            "Run failed with exit status {}: {}",
+            run.exit_status,
+            run.output
+        ))
+    }
+}

xtask/src/main.rs

@@ -0,0 +1,388 @@
+mod build;
+mod command;
+
+use anyhow::bail;
+use core::fmt;
+use std::{
+    error::Error,
+    ffi::OsString,
+    path::{Path, PathBuf},
+    process,
+    process::ExitStatus,
+    str,
+};
+use structopt::StructOpt;
+
+use crate::{
+    build::{build_hexpath, compare_builds, init_build_dir},
+    command::{run_command, run_successful, BuildMode, CargoCommand},
+};
+
+const ARMV6M: &str = "thumbv6m-none-eabi";
+const ARMV7M: &str = "thumbv7m-none-eabi";
+
+#[derive(Debug, StructOpt)]
+struct Options {
+    #[structopt(short, long)]
+    target: String,
+}
+
+#[derive(Debug)]
+pub struct RunResult {
+    exit_status: ExitStatus,
+    output: String,
+}
+
+#[derive(Debug)]
+enum TestRunError {
+    FileCmpError { file_1: String, file_2: String },
+    PathConversionError(OsString),
+    CommandError(RunResult),
+    IncompatibleCommand,
+}
+
+impl fmt::Display for TestRunError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            TestRunError::FileCmpError { file_1, file_2 } => {
+                write!(f, "Differing output in Files: {} {}", file_1, file_2)
+            }
+            TestRunError::CommandError(e) => {
+                write!(
+                    f,
+                    "Command failed with exit status {}: {}",
+                    e.exit_status, e.output
+                )
+            }
+            TestRunError::PathConversionError(p) => {
+                write!(f, "Can't convert path from `OsString` to `String`: {:?}", p)
+            }
+            TestRunError::IncompatibleCommand => {
+                write!(f, "Can't run that command in this context")
+            }
+        }
+    }
+}
+
+impl Error for TestRunError {}
+
+fn main() -> anyhow::Result<()> {
+    // if there's an `xtask` folder, we're *probably* at the root of this repo (we can't just
+    // check the name of `env::current_dir()` because people might clone it into a different name)
+    let probably_running_from_repo_root = Path::new("./xtask").exists();
+    if probably_running_from_repo_root == false {
+        bail!("xtasks can only be executed from the root of the `cortex-m-rtic` repository");
+    }
+
+    let targets = [ARMV7M, ARMV6M];
+    let examples = &[
+        "idle",
+        "init",
+        "hardware",
+        "preempt",
+        "binds",
+        "resource",
+        "lock",
+        "multilock",
+        "only-shared-access",
+        "task",
+        "message",
+        "capacity",
+        "not-sync",
+        "generics",
+        "pool",
+        "ramfunc",
+        "peripherals-taken",
+    ];
+
+    let opts = Options::from_args();
+    let target = &opts.target;
+
+    init_build_dir()?;
+
+    if target == "all" {
+        for t in targets {
+            run_test(t, examples)?;
+            build_test(t, examples)?;
+        }
+    } else if targets.contains(&target.as_str()) {
+        run_test(&target, examples)?;
+        build_test(&target, examples)?;
+    } else {
+        eprintln!(
+            "The target you specified is not available. Available targets are:\
+                    \n{:?}\n\
+                    as well as `all` (testing on all of the above)",
+            targets
+        );
+        process::exit(1);
+    }
+
+    Ok(())
+}
+
+fn run_test(target: &str, examples: &[&str]) -> anyhow::Result<()> {
+    for example in examples {
+        match *example {
+            "pool" => {
+                if target != ARMV6M {
+                    // check this one manually because addresses printed in `pool.run` may vary
+                    let features_v7 = Some("__v7");
+
+                    let debug_run_result = run_command(&CargoCommand::Run {
+                        example,
+                        target,
+                        features: features_v7,
+                        mode: BuildMode::Debug,
+                    })?;
+
+                    if debug_run_result.exit_status.success() {
+                        print_from_output("foo(0x2", &debug_run_result.output);
+                        print_from_output("bar(0x2", &debug_run_result.output);
+                    }
+
+                    let hexpath = &build_hexpath(*example, features_v7, BuildMode::Debug, 1)?;
+
+                    run_command(&CargoCommand::Objcopy {
+                        example,
+                        target,
+                        features: features_v7,
+                        ihex: hexpath,
+                    })?;
+
+                    let release_run_result = run_command(&CargoCommand::Run {
+                        example,
+                        target,
+                        features: features_v7,
+                        mode: BuildMode::Release,
+                    })?;
+
+                    if release_run_result.exit_status.success() {
+                        print_from_output("foo(0x2", &release_run_result.output);
+                        print_from_output("bar(0x2", &release_run_result.output);
+                    }
+
+                    let hexpath = &build_hexpath(*example, features_v7, BuildMode::Release, 1)?;
+                    run_command(&CargoCommand::Objcopy {
+                        example,
+                        target,
+                        features: features_v7,
+                        ihex: hexpath,
+                    })?;
+                }
+            }
+            "types" => {
+                let features_v7 = Some("__v7");
+
+                // TODO this example doesn't exist anymore, can we remove this case?
+                if target != ARMV6M {
+                    arm_example(
+                        &CargoCommand::Run {
+                            example,
+                            target,
+                            features: features_v7,
+                            mode: BuildMode::Debug,
+                        },
+                        1,
+                    )?;
+                    arm_example(
+                        &CargoCommand::Run {
+                            example,
+                            target,
+                            features: features_v7,
+                            mode: BuildMode::Release,
+                        },
+                        1,
+                    )?;
+                }
+            }
+            _ => {
+                arm_example(
+                    &CargoCommand::Run {
+                        example,
+                        target,
+                        features: None,
+                        mode: BuildMode::Debug,
+                    },
+                    1,
+                )?;
+
+                if *example == "types" {
+                    arm_example(
+                        &CargoCommand::Run {
+                            example,
+                            target,
+                            features: None,
+                            mode: BuildMode::Release,
+                        },
+                        1,
+                    )?;
+                } else {
+                    arm_example(
+                        &CargoCommand::Build {
+                            example,
+                            target,
+                            features: None,
+                            mode: BuildMode::Release,
+                        },
+                        1,
+                    )?;
+                }
+            }
+        }
+    }
+
+    Ok(())
+}
+
+// run example binary `example`
+fn arm_example(command: &CargoCommand, build_num: u32) -> anyhow::Result<()> {
+    match *command {
+        CargoCommand::Run {
+            example,
+            target,
+            features,
+            mode,
+        }
+        | CargoCommand::Build {
+            example,
+            target,
+            features,
+            mode,
+        } => {
+            let run_file = format!("{}.run", example);
+            let expected_output_file = ["ci", "expected", &run_file]
+                .iter()
+                .collect::<PathBuf>()
+                .into_os_string()
+                .into_string()
+                .map_err(|e| TestRunError::PathConversionError(e))?;
+
+            // command is either build or run
+            let cargo_run_result = run_command(&command)?;
+            println!("{}", cargo_run_result.output);
+
+            match &command {
+                CargoCommand::Run { .. } => {
+                    if run_successful(&cargo_run_result, expected_output_file).is_err() {
+                        return Err(anyhow::Error::new(TestRunError::CommandError(
+                            cargo_run_result,
+                        )));
+                    }
+                }
+                _ => (),
+            }
+
+            // now, prepare to objcopy
+            let hexpath = build_hexpath(example, features, mode, build_num)?;
+
+            run_command(&CargoCommand::Objcopy {
+                example,
+                target,
+                features,
+                ihex: &hexpath,
+            })?;
+
+            Ok(())
+        }
+        _ => Err(anyhow::Error::new(TestRunError::IncompatibleCommand)),
+    }
+}
+
+fn build_test(target: &str, examples: &[&str]) -> anyhow::Result<()> {
+    run_command(&CargoCommand::Clean)?;
+
+    let mut built = vec![];
+    let build_path: PathBuf = ["target", target, "debug", "examples"].iter().collect();
+
+    for example in examples {
+        match *example {
+            "pool" | "types" => {
+                if target != ARMV6M {
+                    let features_v7 = Some("__v7");
+
+                    arm_example(
+                        &CargoCommand::Build {
+                            target,
+                            example,
+                            mode: BuildMode::Debug,
+                            features: features_v7,
+                        },
+                        2,
+                    )?;
+                    let file_1 = build_hexpath(example, features_v7, BuildMode::Debug, 1)?;
+                    let file_2 = build_hexpath(example, features_v7, BuildMode::Debug, 2)?;
+
+                    compare_builds(file_1, file_2)?;
+
+                    arm_example(
+                        &CargoCommand::Build {
+                            target,
+                            example,
+                            mode: BuildMode::Release,
+                            features: features_v7,
+                        },
+                        2,
+                    )?;
+                    let file_1 = build_hexpath(example, features_v7, BuildMode::Release, 1)?;
+                    let file_2 = build_hexpath(example, features_v7, BuildMode::Release, 2)?;
+
+                    compare_builds(file_1, file_2)?;
+
+                    built.push(build_path.join(example));
+                }
+            }
+            _ => {
+                let no_features = None;
+                arm_example(
+                    &CargoCommand::Build {
+                        target,
+                        example,
+                        mode: BuildMode::Debug,
+                        features: no_features,
+                    },
+                    2,
+                )?;
+                let file_1 = build_hexpath(example, no_features, BuildMode::Debug, 1)?;
+                let file_2 = build_hexpath(example, no_features, BuildMode::Debug, 2)?;
+
+                compare_builds(file_1, file_2)?;
+
+                arm_example(
+                    &CargoCommand::Build {
+                        target,
+                        example,
+                        mode: BuildMode::Release,
+                        features: no_features,
+                    },
+                    2,
+                )?;
+                let file_1 = build_hexpath(example, no_features, BuildMode::Release, 1)?;
+                let file_2 = build_hexpath(example, no_features, BuildMode::Release, 2)?;
+
+                compare_builds(file_1, file_2)?;
+
+                built.push(build_path.join(example));
+            }
+        }
+    }
+
+    let example_paths: Vec<&Path> = built.iter().map(|p| p.as_path()).collect();
+    let size_run_result = run_command(&CargoCommand::Size { example_paths })?;
+
+    if size_run_result.exit_status.success() {
+        println!("{}", size_run_result.output);
+    }
+
+    Ok(())
+}
+
+/// Check if lines in `output` contain `pattern` and print matching lines
+fn print_from_output(pattern: &str, lines: &str) {
+    let lines = lines.split("\n");
+    for line in lines {
+        if line.contains(pattern) {
+            println!("{}", line);
+        }
+    }
+}