Introduce wasm_pages_to_bytes; rename max_heap_size to max_heap_pages_count (#136)

This commit is contained in:
Mike Voronov 2021-12-15 11:12:08 +03:00 committed by GitHub
parent 5e07fe0876
commit 7b0686ab2e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 52 additions and 50 deletions

18
Cargo.lock generated
View File

@ -767,9 +767,9 @@ dependencies = [
[[package]] [[package]]
name = "fluence-app-service" name = "fluence-app-service"
version = "0.11.0" version = "0.11.1"
dependencies = [ dependencies = [
"fluence-faas 0.10.0", "fluence-faas 0.10.1",
"log", "log",
"maplit", "maplit",
"serde", "serde",
@ -807,7 +807,7 @@ dependencies = [
[[package]] [[package]]
name = "fluence-faas" name = "fluence-faas"
version = "0.10.0" version = "0.10.1"
dependencies = [ dependencies = [
"bytesize", "bytesize",
"cmd_lib", "cmd_lib",
@ -817,8 +817,8 @@ dependencies = [
"marine-module-interface 0.1.6", "marine-module-interface 0.1.6",
"marine-rs-sdk", "marine-rs-sdk",
"marine-rs-sdk-main", "marine-rs-sdk-main",
"marine-runtime 0.8.0", "marine-runtime 0.8.1",
"marine-utils 0.3.0", "marine-utils 0.4.0",
"once_cell", "once_cell",
"pretty_assertions", "pretty_assertions",
"safe-transmute", "safe-transmute",
@ -1706,7 +1706,7 @@ dependencies = [
[[package]] [[package]]
name = "marine-runtime" name = "marine-runtime"
version = "0.8.0" version = "0.8.1"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"boolinator", "boolinator",
@ -1718,7 +1718,7 @@ dependencies = [
"marine-it-parser 0.6.8", "marine-it-parser 0.6.8",
"marine-module-info-parser 0.2.2", "marine-module-info-parser 0.2.2",
"marine-module-interface 0.1.6", "marine-module-interface 0.1.6",
"marine-utils 0.3.0", "marine-utils 0.4.0",
"multimap", "multimap",
"once_cell", "once_cell",
"parity-wasm 0.42.2", "parity-wasm 0.42.2",
@ -1793,7 +1793,7 @@ checksum = "8dc5838acba84ce4d802d672afd0814fae0ae7098021ae5b06d975e70d09f812"
[[package]] [[package]]
name = "marine-utils" name = "marine-utils"
version = "0.3.0" version = "0.4.0"
[[package]] [[package]]
name = "matches" name = "matches"
@ -1913,7 +1913,7 @@ dependencies = [
"check-latest", "check-latest",
"clap", "clap",
"env_logger 0.7.1", "env_logger 0.7.1",
"fluence-app-service 0.11.0", "fluence-app-service 0.11.1",
"itertools 0.9.0", "itertools 0.9.0",
"log", "log",
"marine-rs-sdk-main", "marine-rs-sdk-main",

View File

@ -1,7 +1,7 @@
[package] [package]
name = "marine-utils" name = "marine-utils"
description = "Fluence Marine utils crate" description = "Fluence Marine utils crate"
version = "0.3.0" version = "0.4.0"
authors = ["Fluence Labs"] authors = ["Fluence Labs"]
license = "Apache-2.0" license = "Apache-2.0"
edition = "2018" edition = "2018"

View File

@ -16,10 +16,14 @@
pub const WASM_PAGE_SIZE: u32 = 65356; pub const WASM_PAGE_SIZE: u32 = 65356;
pub fn to_wasm_page_count_ceil(offset: u32) -> u32 { pub fn bytes_to_wasm_pages_ceil(offset: u32) -> u32 {
match offset { match offset {
0 => 0, 0 => 0,
// ceiling // ceiling
n => 1 + (n - 1) / WASM_PAGE_SIZE, n => 1 + (n - 1) / WASM_PAGE_SIZE,
} }
} }
pub fn wasm_pages_to_bytes(pages_count: u32) -> u64 {
(pages_count as u64) * (WASM_PAGE_SIZE as u64)
}

View File

@ -1,13 +1,13 @@
[package] [package]
name = "fluence-app-service" name = "fluence-app-service"
description = "Fluence Application Service" description = "Fluence Application Service"
version = "0.11.0" version = "0.11.1"
authors = ["Fluence Labs"] authors = ["Fluence Labs"]
license = "Apache-2.0" license = "Apache-2.0"
edition = "2018" edition = "2018"
[dependencies] [dependencies]
fluence-faas = { path = "../fluence-faas", version = "0.10.0" } fluence-faas = { path = "../fluence-faas", version = "0.10.1" }
maplit = "1.0.2" maplit = "1.0.2"
log = "0.4.8" log = "0.4.8"

View File

@ -1,15 +1,15 @@
[package] [package]
name = "fluence-faas" name = "fluence-faas"
description = "Fluence FaaS" description = "Fluence FaaS"
version = "0.10.0" version = "0.10.1"
authors = ["Fluence Labs"] authors = ["Fluence Labs"]
license = "Apache-2.0" license = "Apache-2.0"
edition = "2018" edition = "2018"
[dependencies] [dependencies]
marine-runtime = { path = "../runtime", version = "0.8.0" } marine-runtime = { path = "../runtime", version = "0.8.1" }
marine-module-interface = { path = "../crates/module-interface", version = "0.1.6" } marine-module-interface = { path = "../crates/module-interface", version = "0.1.6" }
marine-utils = { path = "../crates/utils", version = "0.3.0" } marine-utils = { path = "../crates/utils", version = "0.4.0" }
marine-rs-sdk-main = { version = "0.6.15", features = ["logger"] } marine-rs-sdk-main = { version = "0.6.15", features = ["logger"] }
marine-rs-sdk = { version = "0.6.15", features = ["logger"] } marine-rs-sdk = { version = "0.6.15", features = ["logger"] }

View File

@ -26,7 +26,7 @@ use crate::host_imports::create_call_parameters_import;
use marine::HostImportDescriptor; use marine::HostImportDescriptor;
use marine::MModuleConfig; use marine::MModuleConfig;
use marine_rs_sdk::CallParameters; use marine_rs_sdk::CallParameters;
use marine_utils::to_wasm_page_count_ceil; use marine_utils::bytes_to_wasm_pages_ceil;
use wasmer_core::import::ImportObject; use wasmer_core::import::ImportObject;
use wasmer_core::import::Namespace; use wasmer_core::import::Namespace;
use wasmer_runtime::func; use wasmer_runtime::func;
@ -127,7 +127,7 @@ impl MModuleConfigBuilder {
mem_pages_count: Option<u32>, mem_pages_count: Option<u32>,
max_heap_size: Option<u64>, max_heap_size: Option<u64>,
) -> FaaSResult<Self> { ) -> FaaSResult<Self> {
let mem_pages_count = match (mem_pages_count, max_heap_size) { let max_heap_pages_count = match (mem_pages_count, max_heap_size) {
(Some(v), None) => v, (Some(v), None) => v,
(_, Some(max_heap_size_wanted)) => { (_, Some(max_heap_size_wanted)) => {
if max_heap_size_wanted > WASM_MAX_HEAP_SIZE { if max_heap_size_wanted > WASM_MAX_HEAP_SIZE {
@ -136,13 +136,13 @@ impl MModuleConfigBuilder {
max_heap_size_allowed: WASM_MAX_HEAP_SIZE, max_heap_size_allowed: WASM_MAX_HEAP_SIZE,
}); });
}; };
to_wasm_page_count_ceil(max_heap_size_wanted as u32) bytes_to_wasm_pages_ceil(max_heap_size_wanted as u32)
} }
// leave the default value // leave the default value
(None, None) => return Ok(self), (None, None) => return Ok(self),
}; };
self.config.max_heap_size = mem_pages_count; self.config.max_heap_pages_count = max_heap_pages_count;
Ok(self) Ok(self)
} }

View File

@ -1,7 +1,7 @@
[package] [package]
name = "marine-runtime" name = "marine-runtime"
description = "Marine is the Fluence Compute Runtime" description = "Marine is the Fluence Compute Runtime"
version = "0.8.0" version = "0.8.1"
authors = ["Fluence Labs"] authors = ["Fluence Labs"]
license = "Apache-2.0" license = "Apache-2.0"
edition = "2018" edition = "2018"
@ -16,7 +16,7 @@ marine-it-interfaces = { path = "../crates/it-interfaces", version = "0.4.1" }
marine-it-parser = { path = "../crates/it-parser", version = "0.6.8" } marine-it-parser = { path = "../crates/it-parser", version = "0.6.8" }
marine-it-generator = { path = "../crates/it-generator", version = "0.5.6" } marine-it-generator = { path = "../crates/it-generator", version = "0.5.6" }
marine-module-interface = { path = "../crates/module-interface", version = "0.1.6" } marine-module-interface = { path = "../crates/module-interface", version = "0.1.6" }
marine-utils = { path = "../crates/utils", version = "0.3.0" } marine-utils = { path = "../crates/utils", version = "0.4.0" }
wasmer-runtime = { package = "wasmer-runtime-fl", version = "=0.17.1" } wasmer-runtime = { package = "wasmer-runtime-fl", version = "=0.17.1" }
# dynamicfunc-fat-closures allows using state inside DynamicFunc # dynamicfunc-fat-closures allows using state inside DynamicFunc

View File

@ -26,6 +26,9 @@ use std::path::PathBuf;
use std::collections::HashMap; use std::collections::HashMap;
use std::collections::HashSet; use std::collections::HashSet;
// 65536*1600 ~ 100 Mb (Wasm page size is 64 Kb)
const DEFAULT_HEAP_PAGES_COUNT: u32 = 1600;
pub type HostExportedFunc = Box<dyn Fn(&mut Ctx, Vec<IValue>) -> Option<IValue> + 'static>; pub type HostExportedFunc = Box<dyn Fn(&mut Ctx, Vec<IValue>) -> Option<IValue> + 'static>;
pub struct HostImportDescriptor { pub struct HostImportDescriptor {
@ -46,7 +49,7 @@ pub struct HostImportDescriptor {
pub struct MModuleConfig { pub struct MModuleConfig {
/// Maximum number of Wasm memory pages that loaded module can use. /// Maximum number of Wasm memory pages that loaded module can use.
/// Each Wasm page is 65536 bytes long. /// Each Wasm page is 65536 bytes long.
pub max_heap_size: u32, pub max_heap_pages_count: u32,
/// Import object that will be used in module instantiation process. /// Import object that will be used in module instantiation process.
pub raw_imports: ImportObject, pub raw_imports: ImportObject,
@ -71,8 +74,7 @@ impl Default for MModuleConfig {
fn default() -> Self { fn default() -> Self {
// some reasonable defaults // some reasonable defaults
Self { Self {
// 65536*1600 ~ 100 Mb max_heap_pages_count: DEFAULT_HEAP_PAGES_COUNT,
max_heap_size: 1600,
raw_imports: ImportObject::new(), raw_imports: ImportObject::new(),
host_imports: HashMap::new(), host_imports: HashMap::new(),
wasi_version: WasiVersion::Latest, wasi_version: WasiVersion::Latest,
@ -85,32 +87,28 @@ impl Default for MModuleConfig {
// TODO: implement debug for MModuleConfig // TODO: implement debug for MModuleConfig
impl MModuleConfig {
#[allow(dead_code)] #[allow(dead_code)]
impl MModuleConfig {
pub fn with_mem_pages_count(mut self, mem_pages_count: u32) -> Self { pub fn with_mem_pages_count(mut self, mem_pages_count: u32) -> Self {
self.max_heap_size = mem_pages_count; self.max_heap_pages_count = mem_pages_count;
self self
} }
#[allow(dead_code)]
pub fn with_wasi_version(mut self, wasi_version: WasiVersion) -> Self { pub fn with_wasi_version(mut self, wasi_version: WasiVersion) -> Self {
self.wasi_version = wasi_version; self.wasi_version = wasi_version;
self self
} }
#[allow(dead_code)]
pub fn with_wasi_envs(mut self, envs: HashMap<Vec<u8>, Vec<u8>>) -> Self { pub fn with_wasi_envs(mut self, envs: HashMap<Vec<u8>, Vec<u8>>) -> Self {
self.wasi_envs = envs; self.wasi_envs = envs;
self self
} }
#[allow(dead_code)]
pub fn with_wasi_preopened_files(mut self, preopened_files: HashSet<PathBuf>) -> Self { pub fn with_wasi_preopened_files(mut self, preopened_files: HashSet<PathBuf>) -> Self {
self.wasi_preopened_files = preopened_files; self.wasi_preopened_files = preopened_files;
self self
} }
#[allow(dead_code)]
pub fn with_wasi_mapped_dirs(mut self, mapped_dirs: HashMap<String, PathBuf>) -> Self { pub fn with_wasi_mapped_dirs(mut self, mapped_dirs: HashMap<String, PathBuf>) -> Self {
self.wasi_mapped_dirs = mapped_dirs; self.wasi_mapped_dirs = mapped_dirs;
self self

View File

@ -73,7 +73,8 @@ impl Marine {
wasm_bytes: &[u8], wasm_bytes: &[u8],
config: MModuleConfig, config: MModuleConfig,
) -> MResult<()> { ) -> MResult<()> {
let prepared_wasm_bytes = crate::misc::prepare_module(wasm_bytes, config.max_heap_size)?; let prepared_wasm_bytes =
crate::misc::prepare_module(wasm_bytes, config.max_heap_pages_count)?;
let module = MModule::new(&name, &prepared_wasm_bytes, config, &self.modules)?; let module = MModule::new(&name, &prepared_wasm_bytes, config, &self.modules)?;
match self.modules.entry(name) { match self.modules.entry(name) {

View File

@ -27,10 +27,10 @@ pub enum PrepareError {
#[error(transparent)] #[error(transparent)]
HeapBaseInvalidOrMissing(#[from] HeapBaseError), HeapBaseInvalidOrMissing(#[from] HeapBaseError),
#[error("overflow was happened while summation globals size '{globals_size}' and heap size '{max_heap_size}'")] #[error("overflow was happened while summation globals size '{globals_pages_count}' and heap size '{max_heap_pages_count}'")]
MemSizesOverflow { MemSizesOverflow {
globals_size: u32, globals_pages_count: u32,
max_heap_size: u32, max_heap_pages_count: u32,
}, },
/// Error is encountered while parsing module version. /// Error is encountered while parsing module version.

View File

@ -24,7 +24,7 @@ use super::PrepareResult;
use crate::misc::PrepareError; use crate::misc::PrepareError;
use heap_base::get_heap_base; use heap_base::get_heap_base;
use marine_utils::to_wasm_page_count_ceil; use marine_utils::bytes_to_wasm_pages_ceil;
use parity_wasm::builder; use parity_wasm::builder;
use parity_wasm::elements; use parity_wasm::elements;
@ -43,19 +43,18 @@ impl<'a> ModuleBootstrapper {
Ok(Self { module }) Ok(Self { module })
} }
fn set_max_heap_size(self, max_heap_size: u32) -> PrepareResult<Self> { fn set_max_heap_pages_count(self, max_heap_pages_count: u32) -> PrepareResult<Self> {
use elements::{MemoryType, MemorySection}; use elements::{MemoryType, MemorySection};
let Self { mut module } = self; let Self { mut module } = self;
let globals_size = get_heap_base(&module) let globals_pages_count = get_heap_base(&module)
.map(to_wasm_page_count_ceil) .map(bytes_to_wasm_pages_ceil)
.unwrap_or(DEFAULT_GLOBALS_SIZE); .unwrap_or(DEFAULT_GLOBALS_SIZE);
let max_mem_size = let max_mem_pages_count = globals_pages_count
globals_size .checked_add(max_heap_pages_count)
.checked_add(max_heap_size)
.ok_or(PrepareError::MemSizesOverflow { .ok_or(PrepareError::MemSizesOverflow {
globals_size, globals_pages_count,
max_heap_size, max_heap_pages_count,
})?; })?;
// At now, there is could be only one memory section, so // At now, there is could be only one memory section, so
@ -68,9 +67,9 @@ impl<'a> ModuleBootstrapper {
}, },
None => 0, None => 0,
}; };
let mem_initial_size = std::cmp::min(mem_initial_size, max_mem_size); let mem_initial_size = std::cmp::min(mem_initial_size, max_mem_pages_count);
let memory_entry = MemoryType::new(mem_initial_size, Some(max_mem_size)); let memory_entry = MemoryType::new(mem_initial_size, Some(max_mem_pages_count));
let mut default_mem_section = MemorySection::default(); let mut default_mem_section = MemorySection::default();
module module
@ -94,8 +93,8 @@ impl<'a> ModuleBootstrapper {
/// - extracts __heap_base global /// - extracts __heap_base global
/// - computes module max memory size by summation of heap and globals sizes /// - computes module max memory size by summation of heap and globals sizes
/// - sets computed value as max memory page count of a module /// - sets computed value as max memory page count of a module
pub(crate) fn prepare_module(module: &[u8], max_heap_size: u32) -> PrepareResult<Vec<u8>> { pub(crate) fn prepare_module(module: &[u8], max_heap_pages_count: u32) -> PrepareResult<Vec<u8>> {
ModuleBootstrapper::init(module)? ModuleBootstrapper::init(module)?
.set_max_heap_size(max_heap_size)? .set_max_heap_pages_count(max_heap_pages_count)?
.into_wasm() .into_wasm()
} }