Merge pull request #92 from wasmerio/fix/vm_refactor_esmcripten_integration

[WIP] Fix Emscripten Integration in New Runtime

.gitignore

@@ -2,3 +2,4 @@
 **/*.rs.bk
 /artifacts
 .DS_Store
+.idea

Cargo.lock

@@ -1098,6 +1098,7 @@ dependencies = [
  "time 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
  "wabt 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "wasmer-clif-backend 0.1.0",
+ "wasmer-emscripten 0.1.1",
  "wasmer-runtime 0.1.0",
  "wasmparser 0.23.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1118,6 +1119,20 @@ dependencies = [
  "wasmparser 0.23.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "wasmer-emscripten"
+version = "0.1.1"
+dependencies = [
+ "byteorder 1.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
+ "glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hashbrown 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.44 (git+https://github.com/rust-lang/libc)",
+ "time 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
+ "wabt 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "wasmer-clif-backend 0.1.0",
+ "wasmer-runtime 0.1.0",
+]
+
 [[package]]
 name = "wasmer-runtime"
 version = "0.1.0"

Cargo.toml

@@ -52,11 +52,11 @@ libffi = "0.6.4"
 time = "0.1.41"
 wasmer-clif-backend = { path = "lib/clif-backend" }
 wasmer-runtime = { path = "lib/runtime" }
-# wasmer-emscripten = { path = "lib/emscripten" }
+wasmer-emscripten = { path = "lib/emscripten" }
 libc = { git = "https://github.com/rust-lang/libc" }
 
 [workspace]
-members = ["lib/clif-backend", "lib/runtime"] # "lib/emscripten"
+members = ["lib/clif-backend", "lib/runtime", "lib/emscripten"]
 
 [build-dependencies]
 wabt = "0.7.2"

Makefile

@@ -29,7 +29,7 @@ precommit: lint test
 
 test:
 	# We use one thread so the emscripten stdouts doesn't collide
-	cargo test --all -- --test-threads=1 $(runargs)
+	cargo test --all --exclude wasmer-emscripten -- --test-threads=1 $(runargs)
 
 release:
 	# If you are in OS-X, you will need mingw-w64 for cross compiling to windows

lib/clif-backend/src/module.rs

@@ -32,7 +32,7 @@ impl FuncResolver for PlaceholderFuncResolver {
 
 /// This contains all of the items in a `ModuleInner` except the `func_resolver`.
 pub struct Module {
-    module: ModuleInner,
+    pub module: ModuleInner,
 }
 
 impl Module {

lib/emscripten/Cargo.toml

@@ -6,6 +6,7 @@ edition = "2018"
 build = "build/mod.rs"
 
 [dependencies]
+hashbrown = "0.1"
 wasmer-runtime = { path = "../runtime" }
 libc = { git = "https://github.com/rust-lang/libc" }
 byteorder = "1"

lib/emscripten/src/env.rs

@@ -8,8 +8,8 @@ use std::mem;
 use std::os::raw::c_char;
 
 use super::utils::{allocate_on_stack, copy_cstr_into_wasm, copy_terminated_array_of_cstrs};
-use super::EmscriptenData;
 use wasmer_runtime::{types::Value, Instance};
+//use super::EmscriptenData;
 
 //impl Instance {
 //    pub fn memory_offset_addr(&self, index: usize, offset: usize) -> *const usize {

lib/emscripten/src/syscalls.rs

@@ -68,6 +68,7 @@ use libc::{
     F_GETFD,
     F_SETFD,
     SOL_SOCKET,
+    SO_REUSEADDR,
     TIOCGWINSZ,
 };
 
@@ -336,18 +337,17 @@ pub extern "C" fn ___syscall102(
             unsafe {
                 ioctl(fd, FIOCLEX);
             };
-            if cfg!(target_os = "darwin") {
-                type T = u32;
-                let payload = 1 as *const T as *const c_void;
-                unsafe {
-                    setsockopt(
-                        fd,
-                        SOL_SOCKET,
-                        SO_NOSIGPIPE,
-                        payload,
-                        mem::size_of::<T>() as socklen_t,
-                    );
-                };
+
+            type T = u32;
+            let payload = 1 as *const T as *const c_void;
+            unsafe {
+                setsockopt(
+                    fd,
+                    SOL_SOCKET,
+                    SO_NOSIGPIPE,
+                    payload,
+                    mem::size_of::<T>() as socklen_t,
+                );
             };
 
             debug!(
@@ -364,12 +364,7 @@ pub extern "C" fn ___syscall102(
             let address: u32 = socket_varargs.get(instance);
             let address_len: u32 = socket_varargs.get(instance);
             let address = instance.memory_offset_addr(0, address as usize) as *mut sockaddr;
-            // unsafe {
-            //     debug!(
-            //         "=> address.sin_family: {:?}, address.sin_port: {:?}, address.sin_addr.s_addr: {:?}",
-            //         (*address).sin_family, (*address).sin_port, (*address).sin_addr.s_addr
-            //     );
-            // }
+
             // we convert address as a sockaddr (even if this is incorrect), to bypass the type
             // issue with libc bind
 
@@ -437,25 +432,13 @@ pub extern "C" fn ___syscall102(
                 (*address_linux).sa_family = (*address).sa_family as u16;
                 (*address_linux).sa_data = (*address).sa_data;
             };
-            // // Debug received address
-            // unsafe {
-            //     let proper_address = address as *const GuestSockaddrIn;
-            //     debug!(
-            //         "=> address.sin_family: {:?}, address.sin_port: {:?}, address.sin_addr.s_addr: {:?}",
-            //         (*proper_address).sin_family, (*proper_address).sin_port, (*proper_address).sin_addr.s_addr
-            //     );
-            //     debug!(
-            //         "=> address.sa_family: {:?}",
-            //         (*address).sa_family
-            //     );
-            // }
+
             // set_cloexec
             unsafe {
                 ioctl(fd, FIOCLEX);
             };
             debug!("fd: {}", fd);
-            // nix::unistd::write(fd, "Hello, World!".as_bytes()).unwrap();
-            // nix::unistd::fsync(fd).unwrap();
+
             fd
         }
         6 => {
@@ -515,20 +498,19 @@ pub extern "C" fn ___syscall102(
             //      name: Em passes SO_ACCEPTCONN, but Nginx complains about REUSEADDR
             //      https://github.com/openbsd/src/blob/master/sys/sys/socket.h#L156
             // setsockopt (socket: c_int, level: c_int, name: c_int, value: *const c_void, option_len: socklen_t) -> c_int
+
             let socket: i32 = socket_varargs.get(instance);
-            // SOL_SOCKET = 0xffff in BSD
-            let level: i32 = 0xffff;
+            // SOL_SOCKET = 0xffff (BSD, Linux)
+            let level: i32 = SOL_SOCKET;
             let _: u32 = socket_varargs.get(instance);
-            // SO_ACCEPTCONN = 0x4
-            let name: i32 = 0x4;
+            // SO_REUSEADDR = 0x4 (BSD, Linux)
+            let name: i32 = SO_REUSEADDR;
             let _: u32 = socket_varargs.get(instance);
             let value: u32 = socket_varargs.get(instance);
             let option_len: u32 = socket_varargs.get(instance);
             let value_addr = instance.memory_offset_addr(0, value as usize) as *mut c_void; // Endian problem
             let ret = unsafe { setsockopt(socket, level, name, value_addr, option_len) };
 
-            // debug!("option_value = {:?}", unsafe { *(value_addr as *const u32) });
-
             debug!("=> socketfd: {}, level: {} (SOL_SOCKET/0xffff), name: {} (SO_REUSEADDR/4), value_addr: {:?}, option_len: {} = status: {}", socket, level, name, value_addr, option_len, ret);
             ret
         }

lib/emscripten/src/utils.rs

@@ -6,14 +6,15 @@ use std::ffi::CStr;
 use std::mem::size_of;
 use std::os::raw::c_char;
 use std::slice;
+use std::sync::Arc;
 /// We check if a provided module is an Emscripten generated one
-pub fn is_emscripten_module(module: &Module) -> bool {
-    for (_, import_name) in &module.imported_functions {
-        if import_name.name == "_emscripten_memcpy_big" && import_name.module == "env" {
-            return true;
-        }
-    }
-    false
+pub fn is_emscripten_module(module: &Arc<Module>) -> bool {
+     for (_, import_name) in &module.0.imported_functions {
+         if import_name.name == "_emscripten_memcpy_big" && import_name.namespace == "env" {
+             return true;
+         }
+     }
+     false
 }
 
 pub unsafe fn write_to_buf(string: *const c_char, buf: u32, max: u32, instance: &Instance) -> u32 {
@@ -142,13 +143,19 @@ pub unsafe fn copy_stat_into_wasm(instance: &mut Instance, buf: u32, stat: &stat
 mod tests {
     use super::is_emscripten_module;
     use wasmer_clif_backend::CraneliftCompiler;
+    use wasmer_runtime::{
+        compile,
+        module::Module,
+    };
     use wabt::wat2wasm;
+    use std::sync::Arc;
 
     #[test]
     fn should_detect_emscripten_files() {
         const wast_bytes: &[u8] = include_bytes!("tests/is_emscripten_true.wast");
         let wasm_binary = wat2wasm(wast_bytes.to_vec()).expect("Can't convert to wasm");
-        let module = wasmer_runtime::compile(&wasm_binary[..], &CraneliftCompiler::new()).expect("WASM can't be compiled");
+        let module = compile(&wasm_binary[..], &CraneliftCompiler::new()).expect("WASM can't be compiled");
+        let module = Arc::new(module);
         assert!(is_emscripten_module(&module));
     }
 
@@ -156,7 +163,8 @@ mod tests {
     fn should_detect_non_emscripten_files() {
         const wast_bytes: &[u8] = include_bytes!("tests/is_emscripten_false.wast");
         let wasm_binary = wat2wasm(wast_bytes.to_vec()).expect("Can't convert to wasm");
-        let module = wasmer_runtime::compile(&wasm_binary[..], &CraneliftCompiler::new()).expect("WASM can't be compiled");
+        let module = compile(&wasm_binary[..], &CraneliftCompiler::new()).expect("WASM can't be compiled");
+        let module = Arc::new(module);
         assert!(!is_emscripten_module(&module));
     }
 }

lib/emscripten/tests/emtests/_common.rs

@@ -1,46 +1,55 @@
 macro_rules! assert_emscripten_output {
     ($file:expr, $name:expr, $args:expr, $expected:expr) => {{
-        use wasmer_emscripten::generate_emscripten_env;
-        // use wasmer::common::stdio::StdioCapturer;
-        use wasmer_runtime::{Import, Imports, FuncRef};
-        use wasmer_runtime::table::TableBacking;
-        use wasmer_runtime::{Instance, module::Module};
-        use wasmer_clif_backend::CraneliftCompiler;
 
-        use std::sync::Arc;
+        use wasmer_clif_backend::CraneliftCompiler;
+        use wasmer_emscripten::{
+            EmscriptenGlobals,
+            generate_emscripten_env,
+            stdio::StdioCapturer
+        };
 
         let wasm_bytes = include_bytes!($file);
-        let import_object = generate_emscripten_env();
-//         let options = Some(InstanceOptions {
-//             mock_missing_imports: true,
-//             mock_missing_globals: true,
-//             mock_missing_tables: true,
-//             abi: InstanceABI::Emscripten,
-//             show_progressbar: false,
-// //            isa: get_isa(),
-//         });
-//         let mut result_object = instantiate(&wasm_bytes.to_vec(), &import_object, options)
-//             .expect("Not compiled properly");
 
-        let module = wasmer_runtime::compile(&wasm_bytes[..], &CraneliftCompiler::new()).expect("WASM can't be compiled");
-        let instance = module.instantiate(&import_object).expect("WASM can't be instantiated");
+        let module = wasmer_runtime::compile(&wasm_bytes[..], &CraneliftCompiler::new())
+            .expect("WASM can't be compiled");
+
+//        let module = compile(&wasm_bytes[..])
+//            .map_err(|err| format!("Can't create the WebAssembly module: {}", err)).unwrap(); // NOTE: Need to figure what the unwrap is for ??
+
+        let emscripten_globals = EmscriptenGlobals::new();
+        let import_object = generate_emscripten_env(&emscripten_globals);
+
+        let mut instance = module.instantiate(import_object)
+            .map_err(|err| format!("Can't instantiate the WebAssembly module: {}", err)).unwrap(); // NOTE: Need to figure what the unwrap is for ??
+
+//        start_instance(
+//            Arc::clone(&module),
+//            &mut instance,
+//            $name,
+//            $args,
+//        );
 
-        // let capturer = StdioCapturer::new();
-        // start_instance(
-        //     Arc::clone(&result_object.module),
-        //     &mut result_object.instance,
-        //     $name,
-        //     $args,
-        // )
-        // .unwrap();
-        // let output = capturer.end().unwrap().0;
-        // let expected_output = include_str!($expected);
         assert!(false, "Emscripten tests are mocked");
-        // assert!(
-        //     output.contains(expected_output),
-        //     "Output: `{}` does not contain expected output: `{}`",
-        //     output,
-        //     expected_output
-        // );
+
+         let capturer = StdioCapturer::new();
+
+        instance.call("_main", &[]).map(|_o| ()).unwrap();
+        // TODO handle start instance logic
+//         start_instance(
+//             Arc::clone(&result_object.module),
+//             &mut result_object.instance,
+//             $name,
+//             $args,
+//         )
+//         .unwrap();
+         let output = capturer.end().unwrap().0;
+         let expected_output = include_str!($expected);
+         assert!(false, "Emscripten tests are mocked");
+         assert!(
+             output.contains(expected_output),
+             "Output: `{}` does not contain expected output: `{}`",
+             output,
+             expected_output
+         );
     }};
 }

lib/runtime/src/instance.rs

@@ -24,7 +24,7 @@ pub(crate) struct InstanceInner {
 }
 
 pub struct Instance {
-    pub(crate) module: Rc<ModuleInner>,
+    pub module: Rc<ModuleInner>,
     inner: Box<InstanceInner>,
     #[allow(dead_code)]
     imports: Box<Imports>,
@@ -344,7 +344,7 @@ impl Namespace for Instance {
 
 // TODO Remove this later, only needed for compilation till emscripten is updated
 impl Instance {
-    pub fn memory_offset_addr(&self, _index: usize, _offset: usize) -> *const usize {
-        unimplemented!("TODO replace this emscripten stub")
+    pub fn memory_offset_addr(&self, index: usize, offset: usize) -> *const u8 {
+        unimplemented!()
     }
 }

lib/runtime/src/lib.rs

@@ -3,13 +3,13 @@
 extern crate field_offset;
 
 #[macro_use]
-mod macros;
+pub mod macros;
 #[doc(hidden)]
 pub mod backend;
 mod backing;
 pub mod export;
 pub mod import;
-mod instance;
+pub mod instance;
 pub mod memory;
 mod mmap;
 pub mod module;

lib/runtime/src/macros.rs

@@ -1,3 +1,4 @@
+#[macro_export]
 macro_rules! debug {
     ($fmt:expr) => (if cfg!(any(debug_assertions, feature="debug")) { println!(concat!("wasmer-runtime(:{})::", $fmt), line!()) });
     ($fmt:expr, $($arg:tt)*) => (if cfg!(any(debug_assertions, feature="debug")) { println!(concat!("wasmer-runtime(:{})::", $fmt, "\n"), line!(), $($arg)*) });

lib/runtime/src/module.rs

@@ -39,7 +39,7 @@ pub struct ModuleInner {
     pub sig_registry: SigRegistry,
 }
 
-pub struct Module(Rc<ModuleInner>);
+pub struct Module(pub Rc<ModuleInner>);
 
 impl Module {
     pub(crate) fn new(inner: Rc<ModuleInner>) -> Self {

src/bin/wasmer.rs

@@ -7,12 +7,13 @@ use std::io::Read;
 use std::path::PathBuf;
 use std::process::exit;
 use std::sync::Arc;
+use std::rc::Rc;
 
 use structopt::StructOpt;
 
 use wasmer::*;
 use wasmer_runtime;
-// use wasmer_emscripten;
+use wasmer_emscripten;
 
 #[derive(Debug, StructOpt)]
 #[structopt(name = "wasmer", about = "WASM execution runtime.")]
@@ -74,18 +75,18 @@ fn execute_wasm(options: &Run) -> Result<(), String> {
     let module = webassembly::compile(&wasm_binary[..])
         .map_err(|err| format!("Can't create the WebAssembly module: {}", err))?;
 
-    let (abi, import_object) = if false {
-        // wasmer_emscripten::is_emscripten_module(&module)
-        // (webassembly::InstanceABI::Emscripten, wasmer_emscripten::generate_emscripten_env())
-        (
-            webassembly::InstanceABI::None,
-            wasmer_runtime::Imports::new(),
-        )
+    let abi = if wasmer_emscripten::is_emscripten_module(&module) {
+        webassembly::InstanceABI::Emscripten
     } else {
-        (
-            webassembly::InstanceABI::None,
-            wasmer_runtime::Imports::new(),
-        )
+        webassembly::InstanceABI::None
+    };
+
+    let emscripten_globals = wasmer_emscripten::EmscriptenGlobals::new();
+
+    let mut import_object = if abi == webassembly::InstanceABI::Emscripten {
+        wasmer_emscripten::generate_emscripten_env(&emscripten_globals)
+    } else {
+        wasmer_runtime::import::Imports::new()
     };
 
     let instance_options = webassembly::InstanceOptions {
@@ -94,13 +95,11 @@ fn execute_wasm(options: &Run) -> Result<(), String> {
         mock_missing_tables: true,
         abi: abi,
         show_progressbar: true,
-        //        isa: isa,
     };
 
     debug!("webassembly - creating instance");
 
-    let mut instance = module
-        .instantiate(import_object)
+    let mut instance = module.instantiate(import_object)
         .map_err(|err| format!("Can't instantiate the WebAssembly module: {}", err))?;
 
     webassembly::start_instance(

src/common/mod.rs

@@ -1,6 +1,2 @@
 pub mod mmap;
 pub mod slice;
-
-mod file_descriptor;
-#[cfg(test)]
-pub mod stdio;

src/webassembly/mod.rs

@@ -4,9 +4,12 @@ pub mod relocation;
 pub mod utils;
 
 use wasmer_clif_backend::CraneliftCompiler;
-use wasmer_runtime;
-use wasmer_runtime::{backend::Compiler, module::Module};
-use wasmer_runtime::{Imports, Instance};
+use wasmer_runtime::{
+    backend::Compiler,
+    module::{Module, ModuleInner},
+    import::Imports,
+    instance::Instance,
+};
 
 use cranelift_codegen::{
     isa,
@@ -15,13 +18,14 @@ use cranelift_codegen::{
 use std::panic;
 use std::str::FromStr;
 use std::sync::Arc;
+use std::rc::Rc;
 use target_lexicon;
 use wasmparser;
 use wasmparser::WasmDecoder;
 
 pub use self::errors::{Error, ErrorKind};
 
-// use wasmer_emscripten::{allocate_cstr_on_stack, allocate_on_stack, is_emscripten_module};
+use wasmer_emscripten::{allocate_cstr_on_stack, allocate_on_stack, is_emscripten_module};
 
 pub struct ResultObject {
     /// A webassembly::Module object representing the compiled WebAssembly module.
@@ -118,10 +122,12 @@ pub fn instantiate_streaming(
 /// If the operation fails, the Result rejects with a
 /// webassembly::CompileError.
 pub fn compile(buffer_source: &[u8]) -> Result<Arc<Module>, ErrorKind> {
-    let module = wasmer_runtime::compile(&buffer_source, &CraneliftCompiler::new())
+    let compiler = &CraneliftCompiler {};
+    let module_inner = compiler
+        .compile(buffer_source)
         .map_err(|e| ErrorKind::CompileError(e))?;
 
-    Ok(Arc::new(module))
+    Ok(Arc::new(Module(Rc::new(module_inner))))
 }
 
 /// The webassembly::validate() function validates a given typed
@@ -225,8 +231,7 @@ pub fn start_instance(
     path: &str,
     args: Vec<&str>,
 ) -> Result<(), String> {
-    let main_name = if false {
-        // is_emscripten_module(&instance.module)
+    let main_name = if is_emscripten_module(&module) {
         "_main"
     } else {
         "main"