Merge #268

268: add experimental virtual filesystem r=lachlansneff a=xmclark

This PR introduces some changes to the runtime and adds a new feature: virtual file system.

In the future, we may want to bundle data with our wasm apps. That data can be made available to apps in a virtual filesystem by the wasmer runtime. To do this:

- wasmer must understand custom section data
- wasmer must re-wire emscripten calls to access a virtual file system
- these features are experimental and should be guarded by a feature flag (`--features vfs`)

~This PR adds support for the `read` syscall. ~
Just kidding, this PR does a lot more. Enough syscalls were re-implemented that a code reorg was warranted. 

Additions below ⬇️ 

Several new modules live under and around the `syscalls` module. These features are only supported on linux, but can be expanded to windows.

There are effectively two implementations: one for VFS and one for the host FS. In the future, we will have more tight control over access to the host, but this will do for now. The `select` function was also extracted into a new module as the complexity increased significantly. 

`select`should be refactored in the future to use a runtime like tokio, but that will need to wait.

The introduction of the libsodium dependency introduced some stumbling blocks, so I forked the zbox project and added a new build step. See it [here](https://github.com/wasmerio/zbox/blob/bundle-libsodium/build.rs). This will build libsodium at `cargo build` time and will automatically configure the environment. This will help with adoption and it is something we may be able to give back to zbox.


Additional changes:

After review, this PR is just a bit too large. We will implement the emscripten changes in another PR. This will help accelerate the work on WASI.

Co-authored-by: Mackenzie Clark <mackenzie.a.z.c@gmail.com>

.circleci/config.yml

@@ -1,20 +1,28 @@
+run_with_build_env_vars: &run_with_build_env_vars
+  environment:
+    LLVM_SYS_70_PREFIX: /home/circleci/project/clang+llvm-7.0.0-x86_64-linux-gnu-ubuntu-16.04/
+
+run_install_dependencies: &run_install_dependencies
+  run:
+    name: install dependencies
+    command: |
+      sudo apt-get install -y cmake
+      curl -O https://releases.llvm.org/7.0.0/clang+llvm-7.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz
+      tar xf clang+llvm-7.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz
+
 version: 2
 jobs:
   # Job used for testing
   lint:
     docker:
       - image: circleci/rust:latest
+    <<: *run_with_build_env_vars
     steps:
       - checkout
       - restore_cache:
           keys:
             - v6-lint-{{ arch }}-{{ checksum "Cargo.lock" }}
-      - run:
+      - <<: *run_install_dependencies
-          name: Install dependencies
-          command: |
-            sudo apt-get install -y cmake
-            curl -O https://releases.llvm.org/7.0.0/clang+llvm-7.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz
-            tar xf clang+llvm-7.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz
       - run:
           name: Install lint deps
           command: |
@@ -25,7 +33,6 @@ jobs:
       - run:
           name: Execute lints
           command: |
-            export LLVM_SYS_70_PREFIX="`pwd`/clang+llvm-7.0.0-x86_64-linux-gnu-ubuntu-16.04/"
             make lint
       - save_cache:
           paths:
@@ -33,37 +40,27 @@ jobs:
             - target/debug/.fingerprint
             - target/debug/build
             - target/debug/deps
-          key: v6-lint-{{ arch }}-{{ checksum "Cargo.lock" }}
+          key: v6-test-cargo-cache-linux-{{ arch }}-{{ checksum "Cargo.lock" }}
 
   test:
     docker:
       - image: circleci/rust:latest
+    <<: *run_with_build_env_vars
     steps:
       - checkout
       - restore_cache:
           keys:
             - v6-test-cargo-cache-linux-{{ arch }}-{{ checksum "Cargo.lock" }}
-      - run:
+      - <<: *run_install_dependencies
-          name: Install dependencies
-          command: |
-            sudo apt-get install -y cmake
-            curl -O https://releases.llvm.org/7.0.0/clang+llvm-7.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz
-            tar xf clang+llvm-7.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz
       - run:
           name: Tests
-          command: |
+          command: make test
-            export LLVM_SYS_70_PREFIX="`pwd`/clang+llvm-7.0.0-x86_64-linux-gnu-ubuntu-16.04/"
-            make test
       - run:
           name: Emscripten Tests
-          command: |
+          command: make test-emscripten
-            export LLVM_SYS_70_PREFIX="`pwd`/clang+llvm-7.0.0-x86_64-linux-gnu-ubuntu-16.04/"
-            make test-emscripten
       - run:
           name: Integration Tests
-          command: |
+          command: make integration-tests
-            export LLVM_SYS_70_PREFIX="`pwd`/clang+llvm-7.0.0-x86_64-linux-gnu-ubuntu-16.04/"
-            make integration-tests
       - save_cache:
           paths:
             - /usr/local/cargo/registry

Cargo.toml

@@ -19,27 +19,30 @@ include = [
 ]
 
 [dependencies]
+errno = "0.2.4"
 structopt = "0.2.11"
 wabt = "0.7.2"
 hashbrown = "0.1.8"
 wasmer-clif-backend = { path = "lib/clif-backend" }
+wasmer-dynasm-backend = { path = "lib/dynasm-backend", optional = true }
 wasmer-runtime = { path = "lib/runtime" }
+wasmer-runtime-abi = { path = "lib/runtime-abi", optional = true }
 wasmer-runtime-core = { path = "lib/runtime-core" }
 wasmer-emscripten = { path = "lib/emscripten" }
 wasmer-llvm-backend = { path = "lib/llvm-backend", optional = true }
-wasmer-dynasm-backend = { path = "lib/dynasm-backend", optional = true }
 
 [workspace]
-members = ["lib/clif-backend", "lib/dynasm-backend", "lib/runtime", "lib/runtime-core", "lib/emscripten", "lib/spectests", "lib/win-exception-handler", "lib/runtime-c-api", "lib/llvm-backend"]
+members = ["lib/clif-backend", "lib/dynasm-backend", "lib/runtime", "lib/runtime-abi", "lib/runtime-core", "lib/emscripten", "lib/spectests", "lib/win-exception-handler", "lib/runtime-c-api", "lib/llvm-backend"]
 
 [build-dependencies]
 wabt = "0.7.2"
 glob = "0.2.11"
 
 [features]
-debug = ["wasmer-clif-backend/debug", "wasmer-runtime-core/debug"]
 default = ["fast-tests"]
+debug = ["wasmer-clif-backend/debug", "wasmer-runtime-core/debug"]
 # This feature will allow cargo test to run much faster
 fast-tests = []
 llvm = ["wasmer-llvm-backend"]
-dynasm = ["wasmer-dynasm-backend"]
+dynasm = ["wasmer-dynasm-backend"]
+vfs = ["wasmer-runtime-abi"]

lib/clif-backend/Cargo.toml

@@ -8,7 +8,7 @@ repository = "https://github.com/wasmerio/wasmer"
 edition = "2018"
 
 [dependencies]
-wasmer-runtime-core = { path = "../runtime-core", version = "0.2.0" }
+wasmer-runtime-core = { path = "../runtime-core", version = "0.2.1" }
 cranelift-native = "0.26.0"
 cranelift-codegen = "0.26.0"
 cranelift-entity = "0.26.0"

lib/clif-backend/src/module.rs

@@ -51,6 +51,8 @@ impl Module {
                 namespace_table: StringTable::new(),
                 name_table: StringTable::new(),
                 em_symbol_map: compiler_config.symbol_map.clone(),
+
+                custom_sections: HashMap::new(),
             },
         }
     }

lib/dynasm-backend/Cargo.toml

@@ -16,3 +16,4 @@ lazy_static = "1.2.0"
 byteorder = "1"
 nix = "0.13.0"
 libc = "0.2.49"
+hashbrown = "0.1"

lib/dynasm-backend/src/parse.rs

@@ -1,4 +1,5 @@
 use crate::codegen::{CodegenError, FunctionCodeGenerator, ModuleCodeGenerator};
+use hashbrown::HashMap;
 use wasmer_runtime_core::{
     backend::{Backend, CompilerConfig, FuncResolver, ProtectedCaller},
     module::{
@@ -98,6 +99,8 @@ pub fn read_module<
         name_table: StringTable::new(),
 
         em_symbol_map: compiler_config.symbol_map.clone(),
+
+        custom_sections: HashMap::new(),
     };
 
     let mut reader = ModuleReader::new(wasm)?;

lib/emscripten/emtests/test_vfs.c

@@ -0,0 +1,12 @@
+#include <stdio.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+int main() {
+    char data[256] = {0};
+    ssize_t fd = open("data.txt", 0);
+    ssize_t result = read((int)fd, &data, 255);
+    printf("content: %s", data);
+    printf("fd: %zd\n", fd);
+    return 0;
+}

lib/emscripten/emtests/test_vfs.md

@@ -0,0 +1,6 @@
+The wasm file `test_vfs.wasm` is generated by compiling the `test_vfs.c` and writing a tar.zst blob with a single file 
+named `data.txt`. 
+
+The program expects to find a file named `data.txt` and reads the contents and the file descriptor. 
+
+The runtime should mount the virtual filesystem and expose the file. 

lib/emscripten/emtests/test_vfs.out

@@ -0,0 +1 @@
+content: wasmer is awesomer

lib/emscripten/emtests/test_vfs_data.txt

@@ -0,0 +1 @@
+wasmer is awesomer

lib/emscripten/tests/emtests/test_vfs.rs

@@ -0,0 +1,8 @@
+use crate::emtests::_common::assert_emscripten_output;
+
+#[test]
+fn test_vfs() {
+    let wasm_bytes = include_bytes!("../../emtests/test_vfs_bundle.wasm");
+    let expected_str = include_str!("../../emtests/test_vfs.out");
+    assert_emscripten_output(wasm_bytes, expected_str);
+}

lib/llvm-backend/src/read_info.rs

@@ -18,6 +18,8 @@ use wasmparser::{
     SectionCode, Type as WpType,
 };
 
+use hashbrown::HashMap;
+
 pub fn read_module(
     wasm: &[u8],
     compiler_config: CompilerConfig,
@@ -47,6 +49,8 @@ pub fn read_module(
         name_table: StringTable::new(),
 
         em_symbol_map: compiler_config.symbol_map.clone(),
+
+        custom_sections: HashMap::new(),
     };
 
     let mut reader = ModuleReader::new(wasm)?;

lib/runtime-abi/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+name = "wasmer-runtime-abi"
+version = "0.2.1"
+description = "Wasmer runtime core library"
+license = "MIT"
+authors = ["The Wasmer Engineering Team <engineering@wasmer.io>"]
+repository = "https://github.com/wasmerio/wasmer"
+edition = "2018"
+
+[dependencies]
+libc = "0.2.50"
+wasmer-runtime-core = { path = "../runtime-core" }
+hashbrown = "0.1"
+failure = "0.1"
+tar = "0.4"
+wasmparser = "0.23.0"
+zstd = "0.4"
+
+[target.'cfg(unix)'.dependencies.zbox]
+git = "https://github.com/wasmerio/zbox"
+branch = "bundle-libsodium"
+features = ["libsodium-bundled"]
+
+[dev-dependencies]
+tempdir = "0.3"
+
+[features]
+debug = []

lib/runtime-abi/README.md

@@ -0,0 +1,23 @@
+# runtime-abi
+
+This crate has ABI functions (like syscalls) and extensions to the runtime for enabling ABIs (e.g. virtual filesystem).
+
+## Virtual Filesystem (experimental)
+
+The virtual filesystem allows the runtime to read bundled wasm data as if they were files. Data that is stored in a 
+custom section compressed with [zstd][1] compression and archived with [tar][2] will be exposed as files and mounted
+in the `/` root.
+
+The only current supported operation is the `read` syscall. 
+
+The virtual filesystem is not enabled by default. Build with `--features vfs` to use it. 
+
+[Zbox][3] is a virtual filesystem that depends on [libsodium][4]. See [installation instructions][5] for libsodium here. One can
+statically link libsodium with the [instructions][6] on Zbox's readme. 
+
+[1]: https://facebook.github.io/zstd/
+[2]: https://www.gnu.org/software/tar/
+[3]: https://zbox.io/
+[4]: https://download.libsodium.org/doc/
+[5]: https://download.libsodium.org/doc/installation
+[6]: https://github.com/zboxfs/zbox#static-linking-with-libsodium

lib/runtime-abi/src/lib.rs

@@ -0,0 +1,6 @@
+#[cfg(not(target_os = "windows"))]
+#[macro_use]
+extern crate failure;
+
+#[cfg(not(target_os = "windows"))]
+pub mod vfs;

lib/runtime-abi/src/vfs/device_file.rs

@@ -0,0 +1,111 @@
+use crate::vfs::file_like::{FileLike, Metadata};
+use failure::Error;
+use std::io;
+
+pub struct Stdin;
+pub struct Stdout;
+pub struct Stderr;
+
+impl FileLike for Stdin {
+    fn metadata(&self) -> Result<Metadata, Error> {
+        unimplemented!()
+    }
+
+    fn set_file_len(&mut self, _len: usize) -> Result<(), failure::Error> {
+        panic!("Cannot set length of stdin");
+    }
+}
+
+impl io::Read for Stdin {
+    fn read(&mut self, _buf: &mut [u8]) -> Result<usize, io::Error> {
+        unimplemented!()
+    }
+}
+
+impl io::Write for Stdin {
+    fn write(&mut self, _buf: &[u8]) -> Result<usize, io::Error> {
+        unimplemented!()
+    }
+
+    fn flush(&mut self) -> Result<(), io::Error> {
+        unimplemented!()
+    }
+}
+
+impl io::Seek for Stdin {
+    fn seek(&mut self, _pos: io::SeekFrom) -> Result<u64, io::Error> {
+        unimplemented!()
+    }
+}
+
+impl FileLike for Stdout {
+    fn metadata(&self) -> Result<Metadata, failure::Error> {
+        unimplemented!()
+    }
+
+    fn set_file_len(&mut self, _len: usize) -> Result<(), failure::Error> {
+        panic!("Cannot set length of stdout");
+    }
+}
+
+impl io::Read for Stdout {
+    fn read(&mut self, _buf: &mut [u8]) -> Result<usize, io::Error> {
+        unimplemented!()
+    }
+}
+
+impl io::Write for Stdout {
+    fn write(&mut self, buf: &[u8]) -> Result<usize, io::Error> {
+        let stdout = io::stdout();
+        let mut handle = stdout.lock();
+        handle.write(buf)
+    }
+
+    fn flush(&mut self) -> Result<(), io::Error> {
+        let stdout = io::stdout();
+        let mut handle = stdout.lock();
+        handle.flush()
+    }
+}
+
+impl io::Seek for Stdout {
+    fn seek(&mut self, _pos: io::SeekFrom) -> Result<u64, io::Error> {
+        unimplemented!()
+    }
+}
+
+impl FileLike for Stderr {
+    fn metadata(&self) -> Result<Metadata, failure::Error> {
+        unimplemented!()
+    }
+
+    fn set_file_len(&mut self, _len: usize) -> Result<(), failure::Error> {
+        panic!("Cannot set length of stderr");
+    }
+}
+
+impl io::Read for Stderr {
+    fn read(&mut self, _buf: &mut [u8]) -> Result<usize, io::Error> {
+        unimplemented!()
+    }
+}
+
+impl io::Write for Stderr {
+    fn write(&mut self, buf: &[u8]) -> Result<usize, io::Error> {
+        let stderr = io::stderr();
+        let mut handle = stderr.lock();
+        handle.write(buf)
+    }
+
+    fn flush(&mut self) -> Result<(), io::Error> {
+        let stderr = io::stderr();
+        let mut handle = stderr.lock();
+        handle.flush()
+    }
+}
+
+impl io::Seek for Stderr {
+    fn seek(&mut self, _pos: io::SeekFrom) -> Result<u64, io::Error> {
+        unimplemented!()
+    }
+}

lib/runtime-abi/src/vfs/file_like.rs

@@ -0,0 +1,21 @@
+pub type Fd = isize;
+
+#[derive(Debug)]
+pub struct Metadata {
+    pub len: usize,
+    pub is_file: bool,
+}
+
+pub trait FileLike: std::io::Write + std::io::Read + std::io::Seek {
+    // get metadata
+    fn metadata(&self) -> Result<Metadata, failure::Error>;
+
+    // write
+    //    fn write_file(&mut self, buf: &[u8]) -> Result<usize, io::Error>;
+
+    // read
+    //    fn read_file(&mut self, buf: &mut [u8]) -> Result<usize, io::Error>;
+
+    // set_file_len
+    fn set_file_len(&mut self, len: usize) -> Result<(), failure::Error>;
+}

lib/runtime-abi/src/vfs/mod.rs

@@ -0,0 +1,5 @@
+pub mod device_file;
+pub mod file_like;
+pub mod vfs;
+pub mod vfs_header;
+pub mod virtual_file;

lib/runtime-abi/src/vfs/vfs.rs

@@ -0,0 +1,170 @@
+use crate::vfs::file_like::FileLike;
+use crate::vfs::vfs_header::{header_from_bytes, ArchiveType, CompressionType};
+use crate::vfs::virtual_file::VirtualFile;
+use hashbrown::HashMap;
+use std::cell::RefCell;
+use std::io;
+use std::io::Read;
+use std::path::{Path, PathBuf};
+use std::rc::Rc;
+use tar::EntryType;
+use zbox::{init_env, OpenOptions, Repo, RepoOpener};
+
+pub struct Vfs {
+    repo: Repo,
+    device_files: HashMap<PathBuf, Rc<RefCell<dyn FileLike>>>,
+}
+
+impl Vfs {
+    /// Like `VfsBacking::from_tar_bytes` except it also decompresses from the zstd format.
+    pub fn from_tar_zstd_bytes<Reader: Read>(tar_bytes: Reader) -> Result<Self, failure::Error> {
+        let result = zstd::decode_all(tar_bytes);
+        let decompressed_data = result.unwrap();
+        Self::from_tar_bytes(&decompressed_data[..])
+    }
+
+    /// Match on the type of the compressed-archive and select the correct unpack method
+    pub fn from_compressed_bytes(compressed_data_slice: &[u8]) -> Result<Self, failure::Error> {
+        let data_bytes = &compressed_data_slice[4..];
+        match header_from_bytes(compressed_data_slice)? {
+            (_, CompressionType::ZSTD, ArchiveType::TAR) => Self::from_tar_zstd_bytes(data_bytes),
+            (_, CompressionType::NONE, ArchiveType::TAR) => Self::from_tar_bytes(data_bytes),
+        }
+    }
+
+    /// Create a vfs from raw bytes in tar format
+    pub fn from_tar_bytes<Reader: Read>(tar_bytes: Reader) -> Result<Self, failure::Error> {
+        init_env();
+        let mut repo = RepoOpener::new()
+            .create(true)
+            .open("mem://wasmer_fs", "")
+            .unwrap();
+        let _errors = tar::Archive::new(tar_bytes)
+            .entries()?
+            .map(|entry| {
+                let mut entry: tar::Entry<Reader> = entry?;
+                let path = entry.path()?;
+                let path = convert_to_absolute_path(path);
+                let _result = match (entry.header().entry_type(), path.parent()) {
+                    (EntryType::Regular, Some(parent)) => {
+                        if let Err(e) = repo.create_dir_all(parent) {
+                            if e == zbox::Error::AlreadyExists || e == zbox::Error::IsRoot {
+                            } else {
+                                return Err(VfsAggregateError::ZboxError(e));
+                            }
+                        } else {
+                        }
+                        let mut file = repo.create_file(&path)?;
+                        if entry.header().size().unwrap_or(0) > 0 {
+                            io::copy(&mut entry, &mut file)?;
+                            file.finish()?;
+                        }
+                    }
+                    (EntryType::Directory, _) => {
+                        if let Err(e) = repo.create_dir_all(path) {
+                            if e == zbox::Error::AlreadyExists || e == zbox::Error::IsRoot {
+                            } else {
+                                return Err(VfsAggregateError::ZboxError(e));
+                            }
+                        } else {
+                        }
+                    }
+                    _ => return Err(VfsAggregateError::UnsupportedFileType),
+                };
+                Ok(())
+            })
+            .collect::<Vec<Result<(), VfsAggregateError>>>();
+
+        //        let import_errors = errors.iter().filter_map(|e| e.err()).collect::<Vec<_>>();
+
+        let vfs = Self {
+            repo,
+            device_files: HashMap::new(),
+            //            import_errors: vec![],
+        };
+        Ok(vfs)
+    }
+
+    pub fn new() -> Result<(Self, Vec<VfsError>), failure::Error> {
+        init_env();
+        let repo = RepoOpener::new()
+            .create(true)
+            .open("mem://wasmer_fs", "")
+            .unwrap();
+        Ok((
+            Vfs {
+                repo,
+                device_files: HashMap::new(),
+            },
+            vec![],
+        ))
+    }
+
+    pub fn open_file<P: AsRef<Path>>(&mut self, path: P) -> Option<Rc<RefCell<dyn FileLike>>> {
+        init_env();
+        let path = convert_to_absolute_path(path);
+        if let Ok(file) = OpenOptions::new().write(true).open(&mut self.repo, &path) {
+            Some(Rc::new(RefCell::new(VirtualFile::new(file))))
+        } else if let Some(dev_file) = self.device_files.get(&path) {
+            Some(dev_file.clone())
+        } else {
+            None
+        }
+    }
+
+    pub fn make_dir<P: AsRef<Path>>(&mut self, path: P) {
+        self.repo.create_dir_all(path).unwrap();
+    }
+
+    pub fn create_device_file<P: AsRef<Path>>(&mut self, path: P, file: Rc<RefCell<dyn FileLike>>) {
+        self.device_files.insert(path.as_ref().to_path_buf(), file);
+    }
+}
+
+fn convert_to_absolute_path<P: AsRef<Path>>(path: P) -> PathBuf {
+    let path = path.as_ref();
+    if path.is_relative() {
+        std::path::PathBuf::from("/").join(path)
+    } else {
+        path.to_path_buf()
+    }
+}
+
+pub type Handle = i32;
+#[derive(Debug, Fail)]
+pub enum VfsError {
+    #[fail(display = "File with file descriptor \"{}\" does not exist.", _0)]
+    FileWithFileDescriptorNotExist(Handle),
+    #[fail(display = "File descriptor does not exist.")]
+    FileDescriptorNotExist(Handle),
+    #[fail(display = "Source file descriptor does not exist.")]
+    SourceFileDescriptorDoesNotExist,
+    #[fail(display = "Target file descriptor already exists.")]
+    TargetFileDescriptorAlreadyExists,
+    #[fail(display = "Could not get a mutable reference to the file because it is in use.")]
+    CouldNotGetMutableReferenceToFile,
+}
+
+#[derive(Debug, Fail)]
+pub enum VfsAggregateError {
+    #[fail(display = "Entry error.")]
+    EntryError(std::io::Error),
+    #[fail(display = "IO error.")]
+    IoError(std::io::Error),
+    #[fail(display = "Zbox error.")]
+    ZboxError(zbox::Error),
+    #[fail(display = "Unsupported file type.")]
+    UnsupportedFileType,
+}
+
+impl std::convert::From<std::io::Error> for VfsAggregateError {
+    fn from(error: std::io::Error) -> VfsAggregateError {
+        VfsAggregateError::EntryError(error)
+    }
+}
+
+impl std::convert::From<zbox::Error> for VfsAggregateError {
+    fn from(error: zbox::Error) -> VfsAggregateError {
+        VfsAggregateError::ZboxError(error)
+    }
+}

lib/runtime-abi/src/vfs/vfs_header.rs

@@ -0,0 +1,57 @@
+/// Represents the version of this header schema.
+#[repr(u8)]
+#[derive(Debug, PartialEq)]
+pub enum HeaderVersion {
+    Version1 = 1,
+}
+
+/// Represents the compression type of the file data. Only Zstd or no-compression is supported.
+#[repr(u8)]
+#[derive(Debug, PartialEq)]
+pub enum CompressionType {
+    NONE = 0,
+    ZSTD = 1,
+}
+
+/// Represents the type of archive. The only supported archive is the Tar format.
+#[repr(u8)]
+#[derive(Debug, PartialEq)]
+pub enum ArchiveType {
+    TAR = 0,
+}
+
+// extract the header data from bytes
+pub fn header_from_bytes(
+    bytes: &[u8],
+) -> Result<(HeaderVersion, CompressionType, ArchiveType), HeaderError> {
+    if let Some(bytes) = bytes.get(..4) {
+        let version = match bytes[0] {
+            1 => HeaderVersion::Version1,
+            x => return Err(HeaderError::UnknownHeaderVersion(x)),
+        };
+        let compression_type = match bytes[1] {
+            0 => CompressionType::NONE,
+            1 => CompressionType::ZSTD,
+            x => return Err(HeaderError::UnknownCompressionType(x)),
+        };
+        let archive_type = match bytes[2] {
+            0 => ArchiveType::TAR,
+            x => return Err(HeaderError::UnknownArchiveType(x)),
+        };
+        Ok((version, compression_type, archive_type))
+    } else {
+        Err(HeaderError::HeaderTooSmall)
+    }
+}
+
+#[derive(Debug, Fail)]
+pub enum HeaderError {
+    #[fail(display = "The version is not supported: \"{}\"", _0)]
+    UnknownHeaderVersion(u8),
+    #[fail(display = "The compression type is unknown: \"{}\"", _0)]
+    UnknownCompressionType(u8),
+    #[fail(display = "The archive type is unknown: \"{}\"", _0)]
+    UnknownArchiveType(u8),
+    #[fail(display = "The header is too small.")]
+    HeaderTooSmall,
+}

lib/runtime-abi/src/vfs/virtual_file.rs

@@ -0,0 +1,51 @@
+use crate::vfs::file_like::{FileLike, Metadata};
+use failure::Error;
+use std::io;
+
+pub struct VirtualFile(zbox::File);
+
+impl VirtualFile {
+    pub fn new(file: zbox::File) -> Self {
+        VirtualFile(file)
+    }
+}
+
+impl FileLike for VirtualFile {
+    fn metadata(&self) -> Result<Metadata, Error> {
+        self.0
+            .metadata()
+            .map(|m| Metadata {
+                len: m.content_len(),
+                is_file: m.is_file(),
+            })
+            .map_err(|e: zbox::Error| e.into())
+    }
+
+    fn set_file_len(&mut self, len: usize) -> Result<(), failure::Error> {
+        self.0.set_len(len).map_err(|e| e.into())
+    }
+}
+
+impl io::Write for VirtualFile {
+    fn write(&mut self, buf: &[u8]) -> Result<usize, io::Error> {
+        let result = self.0.write(buf)?;
+        self.0.finish().unwrap();
+        Ok(result)
+    }
+
+    fn flush(&mut self) -> Result<(), io::Error> {
+        self.0.flush()
+    }
+}
+
+impl io::Read for VirtualFile {
+    fn read(&mut self, buf: &mut [u8]) -> Result<usize, io::Error> {
+        self.0.read(buf)
+    }
+}
+
+impl io::Seek for VirtualFile {
+    fn seek(&mut self, pos: io::SeekFrom) -> Result<u64, io::Error> {
+        self.0.seek(pos)
+    }
+}

lib/runtime-core/Cargo.toml

@@ -43,8 +43,8 @@ winapi = { version = "0.3", features = ["memoryapi"] }
 [dev-dependencies]
 field-offset = "0.1.1"
 
+[build-dependencies]
+blake2b_simd = "0.4.1"
+
 [features]
 debug = []
-
-[build-dependencies]
-blake2b_simd = "0.4.1"

lib/runtime-core/src/error.rs

@@ -8,6 +8,7 @@ pub type LinkResult<T> = std::result::Result<T, Vec<LinkError>>;
 pub type RuntimeResult<T> = std::result::Result<T, RuntimeError>;
 pub type CallResult<T> = std::result::Result<T, CallError>;
 pub type ResolveResult<T> = std::result::Result<T, ResolveError>;
+pub type ParseResult<T> = std::result::Result<T, ParseError>;
 
 /// This is returned when the chosen compiler is unable to
 /// successfully compile the provided webassembly module into
@@ -445,3 +446,14 @@ impl Into<GrowError> for MemoryProtectionError {
         GrowError::CouldNotProtectMemory(self)
     }
 }
+
+#[derive(Debug)]
+pub enum ParseError {
+    BinaryReadError,
+}
+
+impl From<wasmparser::BinaryReaderError> for ParseError {
+    fn from(_: wasmparser::BinaryReaderError) -> Self {
+        ParseError::BinaryReadError
+    }
+}

lib/runtime-core/src/lib.rs

@@ -69,7 +69,11 @@ pub fn compile_with(
     let token = backend::Token::generate();
     compiler
         .compile(wasm, Default::default(), token)
-        .map(|inner| module::Module::new(Arc::new(inner)))
+        .map(|mut inner| {
+            let inner_info: &mut crate::module::ModuleInfo = &mut inner.info;
+            inner_info.import_custom_sections(wasm).unwrap();
+            module::Module::new(Arc::new(inner))
+        })
 }
 
 /// The same as `compile_with` but changes the compiler behavior

lib/runtime-core/src/module.rs

@@ -59,6 +59,26 @@ pub struct ModuleInfo {
 
     /// Symbol information from emscripten
     pub em_symbol_map: Option<HashMap<u32, String>>,
+
+    pub custom_sections: HashMap<String, Vec<u8>>,
+}
+
+impl ModuleInfo {
+    pub fn import_custom_sections(&mut self, wasm: &[u8]) -> crate::error::ParseResult<()> {
+        let mut parser = wasmparser::ModuleReader::new(wasm)?;
+        while !parser.eof() {
+            let section = parser.read()?;
+            if let wasmparser::SectionCode::Custom { name, kind: _ } = section.code {
+                let mut reader = section.get_binary_reader();
+                let len = reader.bytes_remaining();
+                let bytes = reader.read_bytes(len)?;
+                let data = bytes.to_vec();
+                let name = String::from_utf8_lossy(name).to_string();
+                self.custom_sections.insert(name, data);
+            }
+        }
+        Ok(())
+    }
 }
 
 /// A compiled WebAssembly module.

lib/runtime-core/src/vm.rs

@@ -547,7 +547,7 @@ mod vm_ctx_tests {
         use crate::backend::{
             sys::Memory, Backend, CacheGen, FuncResolver, ProtectedCaller, Token, UserTrapper,
         };
-        use crate::cache::{Error as CacheError, WasmHash};
+        use crate::cache::Error as CacheError;
         use crate::error::RuntimeResult;
         use crate::types::{FuncIndex, LocalFuncIndex, Value};
         use hashbrown::HashMap;
@@ -617,6 +617,8 @@ mod vm_ctx_tests {
                 name_table: StringTable::new(),
 
                 em_symbol_map: None,
+
+                custom_sections: HashMap::new(),
             },
         }
     }

scripts/install_lib_sodium.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+curl -O https://download.libsodium.org/libsodium/releases/libsodium-1.0.17.tar.gz
+tar xf libsodium-1.0.17.tar.gz
+cd libsodium-1.0.17/
+./configure
+make && make check
+sudo make install