Merge branch 'feature/vm_refactor' into fix/vm_refactor_esmcripten_integration

This commit is contained in:
Brandon Fish 2019-01-17 23:20:56 -06:00
commit 321abe6644
17 changed files with 359 additions and 63 deletions

151
.appveyor.yml Normal file
View File

@ -0,0 +1,151 @@
# This appveyor build file is heavily inspired by uutils/coreutils
# https://raw.githubusercontent.com/uutils/coreutils/d0db7bbaa46dabf65b71e3e33b1ed7595aaacc56/.appveyor.yml
branches:
except:
- master
version: "{build} ~ {branch}"
os: Visual Studio 2017
matrix:
allow_failures:
- CHANNEL: nightly
# - ABI: gnu
environment:
matrix:
# minimum version
# - CHANNEL: 1.31.0
# ARCH: i686
# ABI: msvc
# # "msvc" ABI
# - CHANNEL: stable
# ARCH: i686
# ABI: msvc
# - CHANNEL: stable
# ARCH: x86_64
# ABI: msvc
# - CHANNEL: beta
# ARCH: i686
# ABI: msvc
# - CHANNEL: beta
# ARCH: x86_64
# ABI: msvc
# - CHANNEL: nightly
# ARCH: i686
# ABI: msvc
# - CHANNEL: nightly
# ARCH: x86_64
# ABI: msvc
# # "gnu" ABI
# - CHANNEL: stable
# ARCH: i686
# ABI: gnu
# - CHANNEL: stable
# ARCH: x86_64
# ABI: gnu
# - CHANNEL: beta
# ARCH: i686
# ABI: gnu
# - CHANNEL: beta
# ARCH: x86_64
# ABI: gnu
# - CHANNEL: nightly
# ARCH: i686
# ABI: gnu
# - CHANNEL: nightly
# ARCH: x86_64
# ABI: gnu
# * specific gnu compilers
# - CHANNEL: stable
# ARCH: i686
# ABI: gnu
# MINGW_URL: https://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win32/Personal%20Builds/mingw-builds/4.9.2/threads-win32/dwarf/i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z/download
# MINGW_ARCHIVE: i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z
- CHANNEL: stable
ARCH: x86_64
ABI: gnu
MINGW_URL: https://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win64/Personal%20Builds/mingw-builds/7.3.0/threads-posix/seh/x86_64-7.3.0-release-posix-seh-rt_v5-rev0.7z/download#mingw-w64-x86_64-7.3.0-posix-seh.7z
install:
- echo %PATH%
# force branch checkout (if knowable), then reset to the specific commit ## (can be needed for accurate code coverage info)
# * this allows later apps to see the branch name using standard `git branch` operations, yet always builds the correct specific commit
# * ref: <https://github.com/appveyor/ci/issues/1606>[`@`](https://archive.is/RVpnF)
- if DEFINED APPVEYOR_REPO_BRANCH if /I "%APPVEYOR_REPO_SCM%"=="git" ( git checkout "%APPVEYOR_REPO_BRANCH%" & git reset --hard "%APPVEYOR_REPO_COMMIT%" )
# ensure CWD is project main directory
- cd "%APPVEYOR_BUILD_FOLDER%"
# create a working area
- ps: if ( ! $env:CI_TEMP_DIR ) { $env:CI_TEMP_DIR = "${env:TEMP}\${env:APPVEYOR_JOB_ID}" ; mkdir -force $env:CI_TEMP_DIR | out-null }
# rust installation
- set "TARGET=%ARCH%-pc-windows-%ABI%"
# * install `rust` via `rustup`
- appveyor DownloadFile "https://win.rustup.rs/" -FileName "%CI_TEMP_DIR%\rustup-init.exe"
- call "%CI_TEMP_DIR%\rustup-init.exe" -y --default-toolchain %CHANNEL% --default-host %TARGET% --no-modify-path >NUL
- set "PATH=%PATH%;%USERPROFILE%\.cargo\bin"
- ps: $env:TOOLCHAIN = $(rustup show active-toolchain)
- rename "C:\Program Files\Git\usr\bin\sh.exe" sh2.exe
# * set RUST_BACKTRACE for enhanced error messages
- set RUST_BACKTRACE=1
# * show versions
- rustc -vV
- cargo -vV
# finalize FEATURES
- if /i "%CHANNEL%"=="nightly" set "FEATURES=nightly"
# "gnu" ABI setup
# * use the system MinGW/MSYS if we can
- if /i "%ABI%"=="gnu" set MSYS_BINDIR=C:\msys64\usr\bin
- if /i "%ABI%"=="gnu" if /i "%ARCH%"=="i686" set "MSYS_BITS=32"
- if /i "%ABI%"=="gnu" if /i "%ARCH%"=="x86_64" set "MSYS_BITS=64"
- if defined MSYS_BITS set "MSYS_MINGWDIR=C:\msys64\mingw%MSYS_BITS%"
- if defined MSYS_MINGWDIR set "MSYS_BINDIR=C:\msys64\usr\bin"
## * workaround for rust-lang/rust#47048 / rust-lang/rust#53454 ## !maint: remove when resolved
- if /i "%ABI%"=="gnu" if /i "%ARCH%"=="i686" if not DEFINED MINGW_URL set "MINGW_URL=https://downloads.sourceforge.net/project/mingw-w64/Toolchains targetting Win32/Personal Builds/mingw-builds/8.1.0/threads-posix/dwarf/i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z"
- if /i "%ABI%"=="gnu" if /i "%ARCH%"=="x86_64" if not DEFINED MINGW_URL set "MINGW_URL=https://downloads.sourceforge.net/project/mingw-w64/Toolchains targetting Win64/Personal Builds/mingw-builds/8.1.0/threads-posix/seh/x86_64-8.1.0-release-posix-seh-rt_v6-rev0.7z"
##
# * specific MinGW, if specified
- ps: if ( ! $env:MINGW_ARCHIVE -and $env:MINGW_URL ) { $env:MINGW_ARCHIVE = $($([URI]$env:MINGW_URL).fragment).TrimStart('#') }
- ps: if ( ! $env:MINGW_ARCHIVE -and $env:MINGW_URL ) { $env:MINGW_ARCHIVE = $([URI]$env:MINGW_URL).segments[-1] }
- if defined MINGW_ARCHIVE curl --insecure -fsSL "%MINGW_URL%" -o "%CI_TEMP_DIR%\%MINGW_ARCHIVE%"
- if defined MINGW_ARCHIVE mkdir "%CI_TEMP_DIR%\MinGW" >NUL
- if defined MINGW_ARCHIVE 7z x -y "%CI_TEMP_DIR%\%MINGW_ARCHIVE%" -o"%CI_TEMP_DIR%\MinGW" >NUL
- if defined MINGW_ARCHIVE set "MSYS_MINGWDIR=%CI_TEMP_DIR%\MinGW\mingw%MSYS_BITS%"
- if defined MINGW_ARCHIVE set "MSYS_BINDIR=%MSYS_MINGWDIR%\bin"
# * MinGW/MSYS PATH setup
- if defined MSYS_MINGWDIR set PATH=%MSYS_MINGWDIR%\%ARCH%-w64-mingw32\bin;%MSYS_BINDIR%;%PATH%
## * workaround for rust-lang/rust#47048 / rust-lang/rust#53454 ## !maint: remove when resolved
# ** ref: <https://github.com/rust-lang/rust/issues/47048>, <https://github.com/rust-lang/rust/issues/53454>
# ** egs: <https://github.com/pkgw/tectonic/commit/29686db533d8732d7d97fc94270ed33b77f29295>, <https://github.com/rukai/PF_Sandbox/blob/e842613cf9ff102dfb3fbd87381319e6e6dfe3ae/appveyor.yml>
- if /i "%ABI%"=="gnu" rustup install %CHANNEL%-%ARCH%-pc-windows-msvc
- if /i "%ABI%"=="gnu" rustup default %CHANNEL%-%ARCH%-pc-windows-msvc
- if /i "%ABI%"=="gnu" rustup target add %TARGET%
- if /i "%ABI%"=="gnu" rustup show
- if /i "%ABI%"=="gnu" rustc -vV
- ps: $env:TOOLCHAIN = $(rustup show active-toolchain)
# ** copy libs from gcc toolchain to rust toolchain (more specifically, "crt2.o" and "dllcrt2.o" are needed)
- if defined MSYS_MINGWDIR copy /y "%MSYS_MINGWDIR%\%ARCH%-w64-mingw32\lib\*.o" "%USERPROFILE%\.rustup\toolchains\%TOOLCHAIN%\lib\rustlib\%TARGET%\lib" >NUL
##
- if /i "%ABI%"=="gnu" where gcc
- if /i "%ABI%"=="gnu" gcc --version
# "msvc" ABI setup
- if /i "%ABI%" == "msvc" if /i "%ARCH%" == "i686" call "%VS140COMNTOOLS%\..\..\VC\vcvarsall.bat"
- if /i "%ABI%" == "msvc" if /i "%ARCH%" == "x86_64" call "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x64
- if /i "%ABI%" == "msvc" if /i "%ARCH%" == "x86_64" call "%VS140COMNTOOLS%\..\..\VC\vcvarsall.bat" x86_amd64
artifacts:
- path: target\%TARGET%\debug\wasmer.exe
name: wasmer.exe
build_script:
- set BUILD_CMD=cargo +%TOOLCHAIN% build --target=%TARGET%
- echo [ %BUILD_CMD% ] & %BUILD_CMD%
test_script:
- set TEST_CMD=cargo +%TOOLCHAIN% test --target=%TARGET% --no-fail-fast
- echo [ %TEST_CMD% ] & %TEST_CMD%

View File

@ -67,6 +67,12 @@ jobs:
sudo sysctl -w kern.maxfiles=655360 kern.maxfilesperproc=327680
make test
make lint
- run:
name: Execute integration tests
command: |
export PATH="`pwd`/cmake-3.4.1-Darwin-x86_64/CMake.app/Contents/bin:$PATH"
export PATH="$HOME/.cargo/bin:$PATH"
./integration_tests/nginx/test.sh
test-and-build:
docker:

21
Cargo.lock generated
View File

@ -1097,16 +1097,16 @@ dependencies = [
"tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
"wabt 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
"wasmer-clif-backend 0.1.4",
"wasmer-emscripten 0.1.4",
"wasmer-runtime 0.1.4",
"wasmer-clif-backend 0.1.0",
"wasmer-emscripten 0.1.1",
"wasmer-runtime 0.1.0",
"wasmparser 0.23.0 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "wasmer-clif-backend"
version = "0.1.4"
version = "0.1.0"
dependencies = [
"byteorder 1.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
"cranelift-codegen 0.26.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1115,13 +1115,13 @@ dependencies = [
"cranelift-wasm 0.26.0 (registry+https://github.com/rust-lang/crates.io-index)",
"hashbrown 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
"target-lexicon 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"wasmer-runtime 0.1.4",
"wasmer-runtime 0.1.0",
"wasmparser 0.23.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "wasmer-emscripten"
version = "0.1.4"
version = "0.1.1"
dependencies = [
"byteorder 1.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
"glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1129,14 +1129,13 @@ dependencies = [
"libc 0.2.44 (git+https://github.com/rust-lang/libc)",
"time 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
"wabt 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
"wasmer 0.1.4",
"wasmer-clif-backend 0.1.4",
"wasmer-runtime 0.1.4",
"wasmer-clif-backend 0.1.0",
"wasmer-runtime 0.1.0",
]
[[package]]
name = "wasmer-runtime"
version = "0.1.4"
version = "0.1.0"
dependencies = [
"errno 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)",
"field-offset 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1145,7 +1144,7 @@ dependencies = [
"nix 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
"page_size 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
"wabt 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
"wasmer-clif-backend 0.1.4",
"wasmer-clif-backend 0.1.0",
]
[[package]]

View File

@ -1,7 +1,7 @@
[package]
name = "wasmer"
version = "0.1.4"
authors = ["Syrus Akbary <me@syrusakbary.com>"]
authors = ["The Wasmer Engineering Team <engineering@wasmer.io>"]
edition = "2018"
repository = "https://github.com/wasmerio/wasmer"
publish = true

View File

@ -0,0 +1,12 @@
# `nginx` integration test
This starts wasmer with the nginx wasm file and serves an html
file with some simple text to assert on. The test script does
the assertion.
Run test with:
```
> ./integration_tests/nginx/test.sh
```

View File

@ -0,0 +1 @@
wasmer

View File

@ -0,0 +1 @@
26310

View File

@ -0,0 +1,24 @@
events {
}
# We need this for now, as we want to run nginx as a worker
daemon off;
master_process off;
# We show the errors and info in stderr
error_log /dev/stderr info;
http {
# We show access in the stdout
access_log /dev/stdout;
server {
listen 8080;
server_name _;
location / {
# IMPORTANT: Replace the dir with the one you want to serve (that have an index.html file)
root ./html/;
index index.html;
}
}
}

24
integration_tests/nginx/test.sh Executable file
View File

@ -0,0 +1,24 @@
#! /bin/bash
# Build the release and run nginx
make release
nohup ./target/release/wasmer run examples/nginx/nginx.wasm -- -p integration_tests/nginx/ -c nginx.conf &
sleep 3s
curl localhost:8080 > ./nginx.out
if grep "wasmer" ./nginx.out
then
echo "nginx integration test succeeded"
rm ./nohup.out
rm ./nginx.out
rm -rf ./integration_tests/nginx/*_temp
exit 0
else
echo "nginx integration test failed"
rm ./nohup.out
rm ./nginx.out
rm -rf ./integration_tests/nginx/*_temp
exit -1
fi

View File

@ -1,11 +1,7 @@
[package]
name = "wasmer-clif-backend"
version = "0.1.4"
authors = [
"Lachlan Sneff <lachlan.sneff@gmail.com>",
"Steve Akinyemi <steve@wasmer.io>",
"Syrus Akbary <syrus@wasmer.io>"
]
version = "0.1.0"
authors = ["The Wasmer Engineering Team <engineering@wasmer.io>"]
edition = "2018"
[dependencies]

View File

@ -1,13 +1,7 @@
[package]
name = "wasmer-emscripten"
version = "0.1.4"
authors = [
"Lachlan Sneff <lachlan.sneff@gmail.com>",
"Steve Akinyemi <steve@wasmer.io>",
"Mackenzie Clark <mackenzie@wasmer.io>",
"Brandon Fish <brandon@wasmer.io>",
"Syrus Akbary <syrus@wasmer.io>"
]
version = "0.1.1"
authors = ["The Wasmer Engineering Team <engineering@wasmer.io>"]
edition = "2018"
build = "build/mod.rs"

View File

@ -18,7 +18,6 @@ This process will do something similar to:
```
# Generate the .wasm file
emcc localtime.c -o localtime.js
# Delte the js file, as we don't need it
# Delete the js file, as we don't need it
rm localtime.js
```

View File

@ -1,13 +1,7 @@
[package]
name = "wasmer-runtime"
version = "0.1.4"
authors = [
"Lachlan Sneff <lachlan.sneff@gmail.com>",
"Steve Akinyemi <steve@wasmer.io>",
"Mackenzie Clark <mackenzie@wasmer.io>",
"Brandon Fish <brandon@wasmer.io>",
"Syrus Akbary <syrus@wasmer.io>"
]
version = "0.1.0"
authors = ["The Wasmer Engineering Team <engineering@wasmer.io>"]
edition = "2018"
build = "build/mod.rs"

View File

@ -23,6 +23,7 @@ pub mod vm;
#[doc(hidden)]
pub mod vmcalls;
pub use self::import::Imports;
pub use self::instance::Instance;
#[doc(inline)]
pub use self::module::Module;

View File

@ -2,7 +2,7 @@ pub use crate::{
backing::{ImportBacking, LocalBacking},
types::LocalMemoryIndex,
};
use std::{mem, ptr};
use std::{ffi::c_void, mem, ptr};
#[derive(Debug)]
#[repr(C)]
@ -28,9 +28,13 @@ pub struct Ctx {
/// A pointer to an array of imported functions, indexed by `FuncIndex`.
pub(crate) imported_funcs: *mut ImportedFunc,
/// The parent instance.
/// The local backing of the parent instance.
pub local_backing: *mut LocalBacking,
/// The import backing of the parent instance.
pub import_backing: *mut ImportBacking,
pub data: *mut c_void,
pub data_finalizer: Option<extern "C" fn(data: *mut c_void)>,
}
impl Ctx {
@ -50,6 +54,33 @@ impl Ctx {
local_backing,
import_backing,
data: ptr::null_mut(),
data_finalizer: None,
}
}
pub unsafe fn new_with_data(
local_backing: &mut LocalBacking,
import_backing: &mut ImportBacking,
data: *mut c_void,
data_finalizer: extern "C" fn(*mut c_void),
) -> Self {
Self {
memories: local_backing.vm_memories.as_mut_ptr(),
tables: local_backing.vm_tables.as_mut_ptr(),
globals: local_backing.vm_globals.as_mut_ptr(),
imported_memories: import_backing.memories.as_mut_ptr(),
imported_tables: import_backing.tables.as_mut_ptr(),
imported_globals: import_backing.globals.as_mut_ptr(),
imported_funcs: import_backing.functions.as_mut_ptr(),
local_backing,
import_backing,
data,
data_finalizer: Some(data_finalizer),
}
}
@ -431,3 +462,66 @@ mod vm_offset_tests {
);
}
}
#[cfg(test)]
mod vm_ctx_tests {
use super::{Ctx, ImportBacking, LocalBacking};
use crate::structures::Map;
use std::ffi::c_void;
struct TestData {
x: u32,
y: bool,
str: String,
}
extern "C" fn test_data_finalizer(data: *mut c_void) {
let test_data: &mut TestData = unsafe { &mut *(data as *mut TestData) };
assert_eq!(test_data.x, 10);
assert_eq!(test_data.y, true);
assert_eq!(test_data.str, "Test".to_string());
println!("hello from finalizer");
drop(test_data);
}
#[test]
fn test_callback_on_drop() {
let mut data = TestData {
x: 10,
y: true,
str: "Test".to_string(),
};
let mut local_backing = LocalBacking {
memories: Map::new().into_boxed_map(),
tables: Map::new().into_boxed_map(),
vm_memories: Map::new().into_boxed_map(),
vm_tables: Map::new().into_boxed_map(),
vm_globals: Map::new().into_boxed_map(),
};
let mut import_backing = ImportBacking {
functions: Map::new().into_boxed_map(),
memories: Map::new().into_boxed_map(),
tables: Map::new().into_boxed_map(),
globals: Map::new().into_boxed_map(),
};
let data = &mut data as *mut _ as *mut c_void;
let ctx = unsafe {
Ctx::new_with_data(
&mut local_backing,
&mut import_backing,
data,
test_data_finalizer,
)
};
let ctx_test_data = cast_test_data(ctx.data);
assert_eq!(ctx_test_data.x, 10);
assert_eq!(ctx_test_data.y, true);
assert_eq!(ctx_test_data.str, "Test".to_string());
drop(ctx);
}
fn cast_test_data(data: *mut c_void) -> &'static mut TestData {
let test_data: &mut TestData = unsafe { &mut *(data as *mut TestData) };
test_data
}
}

View File

@ -8,10 +8,10 @@ extern crate libc;
extern crate region;
extern crate structopt;
extern crate wabt;
extern crate wasmparser;
extern crate wasmer_clif_backend;
extern crate wasmer_runtime;
extern crate wasmer_emscripten;
extern crate wasmparser;
// extern crate wasmer_emscripten;
#[macro_use]
extern crate target_lexicon;
extern crate byteorder;

View File

@ -10,6 +10,7 @@ use wasmer_runtime::{
import::Imports,
instance::Instance,
};
use cranelift_codegen::{
isa,
settings::{self, Configurable},
@ -35,7 +36,6 @@ pub struct ResultObject {
pub instance: Box<Instance>,
}
pub struct InstanceOptions {
// Shall we mock automatically the imported functions if they don't exist?
pub mock_missing_imports: bool,
@ -43,7 +43,7 @@ pub struct InstanceOptions {
pub mock_missing_tables: bool,
pub abi: InstanceABI,
pub show_progressbar: bool,
// pub isa: Box<isa::TargetIsa>, TODO isa
// pub isa: Box<isa::TargetIsa>, TODO isa
}
#[derive(PartialEq)]
@ -75,10 +75,10 @@ pub fn instantiate(
//let instance = Instance::new(&module, import_object, options)?;
unimplemented!()
// let instance = wasmer_runtime::instantiate(buffer_source, &CraneliftCompiler::new(), import_object)
// .map_err(|e| ErrorKind::CompileError(e))?;
//
// let isa = get_isa();
// let instance = wasmer_runtime::instantiate(buffer_source, &CraneliftCompiler::new(), import_object)
// .map_err(|e| ErrorKind::CompileError(e))?;
//
// let isa = get_isa();
// let abi = if is_emscripten_module(&instance.module) {
// InstanceABI::Emscripten
// } else {
@ -94,11 +94,11 @@ pub fn instantiate(
// isa,
// });
// debug!("webassembly - instance created");
// Ok(ResultObject {
// module: Arc::clone(&instance.module),
// instance,
// })
// debug!("webassembly - instance created");
// Ok(ResultObject {
// module: Arc::clone(&instance.module),
// instance,
// })
}
/// The webassembly::instantiate_streaming() function compiles and instantiates
@ -173,23 +173,23 @@ pub fn get_isa() -> Box<isa::TargetIsa> {
isa::lookup(triple!("x86_64")).unwrap().finish(flags)
}
fn store_module_arguments(path: &str, args: Vec<&str>, instance: &mut Instance) -> (u32, u32) {
let argc = args.len() + 1;
// fn store_module_arguments(path: &str, args: Vec<&str>, instance: &mut Instance) -> (u32, u32) {
// let argc = args.len() + 1;
let (argv_offset, argv_slice): (_, &mut [u32]) =
unsafe { allocate_on_stack(((argc + 1) * 4) as u32, instance) };
assert!(!argv_slice.is_empty());
// let (argv_offset, argv_slice): (_, &mut [u32]) =
// unsafe { allocate_on_stack(((argc + 1) * 4) as u32, instance) };
// assert!(!argv_slice.is_empty());
argv_slice[0] = unsafe { allocate_cstr_on_stack(path, instance).0 };
// argv_slice[0] = unsafe { allocate_cstr_on_stack(path, instance).0 };
for (slot, arg) in argv_slice[1..argc].iter_mut().zip(args.iter()) {
*slot = unsafe { allocate_cstr_on_stack(&arg, instance).0 };
}
// for (slot, arg) in argv_slice[1..argc].iter_mut().zip(args.iter()) {
// *slot = unsafe { allocate_cstr_on_stack(&arg, instance).0 };
// }
argv_slice[argc] = 0;
// argv_slice[argc] = 0;
(argc as u32, argv_offset)
}
// (argc as u32, argv_offset)
// }
// fn get_module_arguments(options: &Run, instance: &mut webassembly::Instance) -> (u32, u32) {
// // Application Arguments