mirror of
https://github.com/fluencelabs/marine.git
synced 2025-03-15 05:50:49 +00:00
Introduce wasm_pages_to_bytes; rename max_heap_size to max_heap_pages_count (#136)
This commit is contained in:
parent
5e07fe0876
commit
7b0686ab2e
18
Cargo.lock
generated
18
Cargo.lock
generated
@ -767,9 +767,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "fluence-app-service"
|
name = "fluence-app-service"
|
||||||
version = "0.11.0"
|
version = "0.11.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"fluence-faas 0.10.0",
|
"fluence-faas 0.10.1",
|
||||||
"log",
|
"log",
|
||||||
"maplit",
|
"maplit",
|
||||||
"serde",
|
"serde",
|
||||||
@ -807,7 +807,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "fluence-faas"
|
name = "fluence-faas"
|
||||||
version = "0.10.0"
|
version = "0.10.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytesize",
|
"bytesize",
|
||||||
"cmd_lib",
|
"cmd_lib",
|
||||||
@ -817,8 +817,8 @@ dependencies = [
|
|||||||
"marine-module-interface 0.1.6",
|
"marine-module-interface 0.1.6",
|
||||||
"marine-rs-sdk",
|
"marine-rs-sdk",
|
||||||
"marine-rs-sdk-main",
|
"marine-rs-sdk-main",
|
||||||
"marine-runtime 0.8.0",
|
"marine-runtime 0.8.1",
|
||||||
"marine-utils 0.3.0",
|
"marine-utils 0.4.0",
|
||||||
"once_cell",
|
"once_cell",
|
||||||
"pretty_assertions",
|
"pretty_assertions",
|
||||||
"safe-transmute",
|
"safe-transmute",
|
||||||
@ -1706,7 +1706,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "marine-runtime"
|
name = "marine-runtime"
|
||||||
version = "0.8.0"
|
version = "0.8.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"boolinator",
|
"boolinator",
|
||||||
@ -1718,7 +1718,7 @@ dependencies = [
|
|||||||
"marine-it-parser 0.6.8",
|
"marine-it-parser 0.6.8",
|
||||||
"marine-module-info-parser 0.2.2",
|
"marine-module-info-parser 0.2.2",
|
||||||
"marine-module-interface 0.1.6",
|
"marine-module-interface 0.1.6",
|
||||||
"marine-utils 0.3.0",
|
"marine-utils 0.4.0",
|
||||||
"multimap",
|
"multimap",
|
||||||
"once_cell",
|
"once_cell",
|
||||||
"parity-wasm 0.42.2",
|
"parity-wasm 0.42.2",
|
||||||
@ -1793,7 +1793,7 @@ checksum = "8dc5838acba84ce4d802d672afd0814fae0ae7098021ae5b06d975e70d09f812"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "marine-utils"
|
name = "marine-utils"
|
||||||
version = "0.3.0"
|
version = "0.4.0"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "matches"
|
name = "matches"
|
||||||
@ -1913,7 +1913,7 @@ dependencies = [
|
|||||||
"check-latest",
|
"check-latest",
|
||||||
"clap",
|
"clap",
|
||||||
"env_logger 0.7.1",
|
"env_logger 0.7.1",
|
||||||
"fluence-app-service 0.11.0",
|
"fluence-app-service 0.11.1",
|
||||||
"itertools 0.9.0",
|
"itertools 0.9.0",
|
||||||
"log",
|
"log",
|
||||||
"marine-rs-sdk-main",
|
"marine-rs-sdk-main",
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "marine-utils"
|
name = "marine-utils"
|
||||||
description = "Fluence Marine utils crate"
|
description = "Fluence Marine utils crate"
|
||||||
version = "0.3.0"
|
version = "0.4.0"
|
||||||
authors = ["Fluence Labs"]
|
authors = ["Fluence Labs"]
|
||||||
license = "Apache-2.0"
|
license = "Apache-2.0"
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
|
@ -16,10 +16,14 @@
|
|||||||
|
|
||||||
pub const WASM_PAGE_SIZE: u32 = 65356;
|
pub const WASM_PAGE_SIZE: u32 = 65356;
|
||||||
|
|
||||||
pub fn to_wasm_page_count_ceil(offset: u32) -> u32 {
|
pub fn bytes_to_wasm_pages_ceil(offset: u32) -> u32 {
|
||||||
match offset {
|
match offset {
|
||||||
0 => 0,
|
0 => 0,
|
||||||
// ceiling
|
// ceiling
|
||||||
n => 1 + (n - 1) / WASM_PAGE_SIZE,
|
n => 1 + (n - 1) / WASM_PAGE_SIZE,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn wasm_pages_to_bytes(pages_count: u32) -> u64 {
|
||||||
|
(pages_count as u64) * (WASM_PAGE_SIZE as u64)
|
||||||
|
}
|
||||||
|
@ -1,13 +1,13 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "fluence-app-service"
|
name = "fluence-app-service"
|
||||||
description = "Fluence Application Service"
|
description = "Fluence Application Service"
|
||||||
version = "0.11.0"
|
version = "0.11.1"
|
||||||
authors = ["Fluence Labs"]
|
authors = ["Fluence Labs"]
|
||||||
license = "Apache-2.0"
|
license = "Apache-2.0"
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
fluence-faas = { path = "../fluence-faas", version = "0.10.0" }
|
fluence-faas = { path = "../fluence-faas", version = "0.10.1" }
|
||||||
|
|
||||||
maplit = "1.0.2"
|
maplit = "1.0.2"
|
||||||
log = "0.4.8"
|
log = "0.4.8"
|
||||||
|
@ -1,15 +1,15 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "fluence-faas"
|
name = "fluence-faas"
|
||||||
description = "Fluence FaaS"
|
description = "Fluence FaaS"
|
||||||
version = "0.10.0"
|
version = "0.10.1"
|
||||||
authors = ["Fluence Labs"]
|
authors = ["Fluence Labs"]
|
||||||
license = "Apache-2.0"
|
license = "Apache-2.0"
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
marine-runtime = { path = "../runtime", version = "0.8.0" }
|
marine-runtime = { path = "../runtime", version = "0.8.1" }
|
||||||
marine-module-interface = { path = "../crates/module-interface", version = "0.1.6" }
|
marine-module-interface = { path = "../crates/module-interface", version = "0.1.6" }
|
||||||
marine-utils = { path = "../crates/utils", version = "0.3.0" }
|
marine-utils = { path = "../crates/utils", version = "0.4.0" }
|
||||||
marine-rs-sdk-main = { version = "0.6.15", features = ["logger"] }
|
marine-rs-sdk-main = { version = "0.6.15", features = ["logger"] }
|
||||||
marine-rs-sdk = { version = "0.6.15", features = ["logger"] }
|
marine-rs-sdk = { version = "0.6.15", features = ["logger"] }
|
||||||
|
|
||||||
|
@ -26,7 +26,7 @@ use crate::host_imports::create_call_parameters_import;
|
|||||||
use marine::HostImportDescriptor;
|
use marine::HostImportDescriptor;
|
||||||
use marine::MModuleConfig;
|
use marine::MModuleConfig;
|
||||||
use marine_rs_sdk::CallParameters;
|
use marine_rs_sdk::CallParameters;
|
||||||
use marine_utils::to_wasm_page_count_ceil;
|
use marine_utils::bytes_to_wasm_pages_ceil;
|
||||||
use wasmer_core::import::ImportObject;
|
use wasmer_core::import::ImportObject;
|
||||||
use wasmer_core::import::Namespace;
|
use wasmer_core::import::Namespace;
|
||||||
use wasmer_runtime::func;
|
use wasmer_runtime::func;
|
||||||
@ -127,7 +127,7 @@ impl MModuleConfigBuilder {
|
|||||||
mem_pages_count: Option<u32>,
|
mem_pages_count: Option<u32>,
|
||||||
max_heap_size: Option<u64>,
|
max_heap_size: Option<u64>,
|
||||||
) -> FaaSResult<Self> {
|
) -> FaaSResult<Self> {
|
||||||
let mem_pages_count = match (mem_pages_count, max_heap_size) {
|
let max_heap_pages_count = match (mem_pages_count, max_heap_size) {
|
||||||
(Some(v), None) => v,
|
(Some(v), None) => v,
|
||||||
(_, Some(max_heap_size_wanted)) => {
|
(_, Some(max_heap_size_wanted)) => {
|
||||||
if max_heap_size_wanted > WASM_MAX_HEAP_SIZE {
|
if max_heap_size_wanted > WASM_MAX_HEAP_SIZE {
|
||||||
@ -136,13 +136,13 @@ impl MModuleConfigBuilder {
|
|||||||
max_heap_size_allowed: WASM_MAX_HEAP_SIZE,
|
max_heap_size_allowed: WASM_MAX_HEAP_SIZE,
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
to_wasm_page_count_ceil(max_heap_size_wanted as u32)
|
bytes_to_wasm_pages_ceil(max_heap_size_wanted as u32)
|
||||||
}
|
}
|
||||||
// leave the default value
|
// leave the default value
|
||||||
(None, None) => return Ok(self),
|
(None, None) => return Ok(self),
|
||||||
};
|
};
|
||||||
|
|
||||||
self.config.max_heap_size = mem_pages_count;
|
self.config.max_heap_pages_count = max_heap_pages_count;
|
||||||
|
|
||||||
Ok(self)
|
Ok(self)
|
||||||
}
|
}
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "marine-runtime"
|
name = "marine-runtime"
|
||||||
description = "Marine is the Fluence Compute Runtime"
|
description = "Marine is the Fluence Compute Runtime"
|
||||||
version = "0.8.0"
|
version = "0.8.1"
|
||||||
authors = ["Fluence Labs"]
|
authors = ["Fluence Labs"]
|
||||||
license = "Apache-2.0"
|
license = "Apache-2.0"
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
@ -16,7 +16,7 @@ marine-it-interfaces = { path = "../crates/it-interfaces", version = "0.4.1" }
|
|||||||
marine-it-parser = { path = "../crates/it-parser", version = "0.6.8" }
|
marine-it-parser = { path = "../crates/it-parser", version = "0.6.8" }
|
||||||
marine-it-generator = { path = "../crates/it-generator", version = "0.5.6" }
|
marine-it-generator = { path = "../crates/it-generator", version = "0.5.6" }
|
||||||
marine-module-interface = { path = "../crates/module-interface", version = "0.1.6" }
|
marine-module-interface = { path = "../crates/module-interface", version = "0.1.6" }
|
||||||
marine-utils = { path = "../crates/utils", version = "0.3.0" }
|
marine-utils = { path = "../crates/utils", version = "0.4.0" }
|
||||||
|
|
||||||
wasmer-runtime = { package = "wasmer-runtime-fl", version = "=0.17.1" }
|
wasmer-runtime = { package = "wasmer-runtime-fl", version = "=0.17.1" }
|
||||||
# dynamicfunc-fat-closures allows using state inside DynamicFunc
|
# dynamicfunc-fat-closures allows using state inside DynamicFunc
|
||||||
|
@ -26,6 +26,9 @@ use std::path::PathBuf;
|
|||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::collections::HashSet;
|
use std::collections::HashSet;
|
||||||
|
|
||||||
|
// 65536*1600 ~ 100 Mb (Wasm page size is 64 Kb)
|
||||||
|
const DEFAULT_HEAP_PAGES_COUNT: u32 = 1600;
|
||||||
|
|
||||||
pub type HostExportedFunc = Box<dyn Fn(&mut Ctx, Vec<IValue>) -> Option<IValue> + 'static>;
|
pub type HostExportedFunc = Box<dyn Fn(&mut Ctx, Vec<IValue>) -> Option<IValue> + 'static>;
|
||||||
|
|
||||||
pub struct HostImportDescriptor {
|
pub struct HostImportDescriptor {
|
||||||
@ -46,7 +49,7 @@ pub struct HostImportDescriptor {
|
|||||||
pub struct MModuleConfig {
|
pub struct MModuleConfig {
|
||||||
/// Maximum number of Wasm memory pages that loaded module can use.
|
/// Maximum number of Wasm memory pages that loaded module can use.
|
||||||
/// Each Wasm page is 65536 bytes long.
|
/// Each Wasm page is 65536 bytes long.
|
||||||
pub max_heap_size: u32,
|
pub max_heap_pages_count: u32,
|
||||||
|
|
||||||
/// Import object that will be used in module instantiation process.
|
/// Import object that will be used in module instantiation process.
|
||||||
pub raw_imports: ImportObject,
|
pub raw_imports: ImportObject,
|
||||||
@ -71,8 +74,7 @@ impl Default for MModuleConfig {
|
|||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
// some reasonable defaults
|
// some reasonable defaults
|
||||||
Self {
|
Self {
|
||||||
// 65536*1600 ~ 100 Mb
|
max_heap_pages_count: DEFAULT_HEAP_PAGES_COUNT,
|
||||||
max_heap_size: 1600,
|
|
||||||
raw_imports: ImportObject::new(),
|
raw_imports: ImportObject::new(),
|
||||||
host_imports: HashMap::new(),
|
host_imports: HashMap::new(),
|
||||||
wasi_version: WasiVersion::Latest,
|
wasi_version: WasiVersion::Latest,
|
||||||
@ -85,32 +87,28 @@ impl Default for MModuleConfig {
|
|||||||
|
|
||||||
// TODO: implement debug for MModuleConfig
|
// TODO: implement debug for MModuleConfig
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
impl MModuleConfig {
|
impl MModuleConfig {
|
||||||
#[allow(dead_code)]
|
|
||||||
pub fn with_mem_pages_count(mut self, mem_pages_count: u32) -> Self {
|
pub fn with_mem_pages_count(mut self, mem_pages_count: u32) -> Self {
|
||||||
self.max_heap_size = mem_pages_count;
|
self.max_heap_pages_count = mem_pages_count;
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
|
||||||
pub fn with_wasi_version(mut self, wasi_version: WasiVersion) -> Self {
|
pub fn with_wasi_version(mut self, wasi_version: WasiVersion) -> Self {
|
||||||
self.wasi_version = wasi_version;
|
self.wasi_version = wasi_version;
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
|
||||||
pub fn with_wasi_envs(mut self, envs: HashMap<Vec<u8>, Vec<u8>>) -> Self {
|
pub fn with_wasi_envs(mut self, envs: HashMap<Vec<u8>, Vec<u8>>) -> Self {
|
||||||
self.wasi_envs = envs;
|
self.wasi_envs = envs;
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
|
||||||
pub fn with_wasi_preopened_files(mut self, preopened_files: HashSet<PathBuf>) -> Self {
|
pub fn with_wasi_preopened_files(mut self, preopened_files: HashSet<PathBuf>) -> Self {
|
||||||
self.wasi_preopened_files = preopened_files;
|
self.wasi_preopened_files = preopened_files;
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
|
||||||
pub fn with_wasi_mapped_dirs(mut self, mapped_dirs: HashMap<String, PathBuf>) -> Self {
|
pub fn with_wasi_mapped_dirs(mut self, mapped_dirs: HashMap<String, PathBuf>) -> Self {
|
||||||
self.wasi_mapped_dirs = mapped_dirs;
|
self.wasi_mapped_dirs = mapped_dirs;
|
||||||
self
|
self
|
||||||
|
@ -73,7 +73,8 @@ impl Marine {
|
|||||||
wasm_bytes: &[u8],
|
wasm_bytes: &[u8],
|
||||||
config: MModuleConfig,
|
config: MModuleConfig,
|
||||||
) -> MResult<()> {
|
) -> MResult<()> {
|
||||||
let prepared_wasm_bytes = crate::misc::prepare_module(wasm_bytes, config.max_heap_size)?;
|
let prepared_wasm_bytes =
|
||||||
|
crate::misc::prepare_module(wasm_bytes, config.max_heap_pages_count)?;
|
||||||
let module = MModule::new(&name, &prepared_wasm_bytes, config, &self.modules)?;
|
let module = MModule::new(&name, &prepared_wasm_bytes, config, &self.modules)?;
|
||||||
|
|
||||||
match self.modules.entry(name) {
|
match self.modules.entry(name) {
|
||||||
|
@ -27,10 +27,10 @@ pub enum PrepareError {
|
|||||||
#[error(transparent)]
|
#[error(transparent)]
|
||||||
HeapBaseInvalidOrMissing(#[from] HeapBaseError),
|
HeapBaseInvalidOrMissing(#[from] HeapBaseError),
|
||||||
|
|
||||||
#[error("overflow was happened while summation globals size '{globals_size}' and heap size '{max_heap_size}'")]
|
#[error("overflow was happened while summation globals size '{globals_pages_count}' and heap size '{max_heap_pages_count}'")]
|
||||||
MemSizesOverflow {
|
MemSizesOverflow {
|
||||||
globals_size: u32,
|
globals_pages_count: u32,
|
||||||
max_heap_size: u32,
|
max_heap_pages_count: u32,
|
||||||
},
|
},
|
||||||
|
|
||||||
/// Error is encountered while parsing module version.
|
/// Error is encountered while parsing module version.
|
||||||
|
@ -24,7 +24,7 @@ use super::PrepareResult;
|
|||||||
use crate::misc::PrepareError;
|
use crate::misc::PrepareError;
|
||||||
use heap_base::get_heap_base;
|
use heap_base::get_heap_base;
|
||||||
|
|
||||||
use marine_utils::to_wasm_page_count_ceil;
|
use marine_utils::bytes_to_wasm_pages_ceil;
|
||||||
use parity_wasm::builder;
|
use parity_wasm::builder;
|
||||||
use parity_wasm::elements;
|
use parity_wasm::elements;
|
||||||
|
|
||||||
@ -43,20 +43,19 @@ impl<'a> ModuleBootstrapper {
|
|||||||
Ok(Self { module })
|
Ok(Self { module })
|
||||||
}
|
}
|
||||||
|
|
||||||
fn set_max_heap_size(self, max_heap_size: u32) -> PrepareResult<Self> {
|
fn set_max_heap_pages_count(self, max_heap_pages_count: u32) -> PrepareResult<Self> {
|
||||||
use elements::{MemoryType, MemorySection};
|
use elements::{MemoryType, MemorySection};
|
||||||
|
|
||||||
let Self { mut module } = self;
|
let Self { mut module } = self;
|
||||||
let globals_size = get_heap_base(&module)
|
let globals_pages_count = get_heap_base(&module)
|
||||||
.map(to_wasm_page_count_ceil)
|
.map(bytes_to_wasm_pages_ceil)
|
||||||
.unwrap_or(DEFAULT_GLOBALS_SIZE);
|
.unwrap_or(DEFAULT_GLOBALS_SIZE);
|
||||||
let max_mem_size =
|
let max_mem_pages_count = globals_pages_count
|
||||||
globals_size
|
.checked_add(max_heap_pages_count)
|
||||||
.checked_add(max_heap_size)
|
.ok_or(PrepareError::MemSizesOverflow {
|
||||||
.ok_or(PrepareError::MemSizesOverflow {
|
globals_pages_count,
|
||||||
globals_size,
|
max_heap_pages_count,
|
||||||
max_heap_size,
|
})?;
|
||||||
})?;
|
|
||||||
|
|
||||||
// At now, there is could be only one memory section, so
|
// At now, there is could be only one memory section, so
|
||||||
// it needs just to extract previous initial page count,
|
// it needs just to extract previous initial page count,
|
||||||
@ -68,9 +67,9 @@ impl<'a> ModuleBootstrapper {
|
|||||||
},
|
},
|
||||||
None => 0,
|
None => 0,
|
||||||
};
|
};
|
||||||
let mem_initial_size = std::cmp::min(mem_initial_size, max_mem_size);
|
let mem_initial_size = std::cmp::min(mem_initial_size, max_mem_pages_count);
|
||||||
|
|
||||||
let memory_entry = MemoryType::new(mem_initial_size, Some(max_mem_size));
|
let memory_entry = MemoryType::new(mem_initial_size, Some(max_mem_pages_count));
|
||||||
let mut default_mem_section = MemorySection::default();
|
let mut default_mem_section = MemorySection::default();
|
||||||
|
|
||||||
module
|
module
|
||||||
@ -94,8 +93,8 @@ impl<'a> ModuleBootstrapper {
|
|||||||
/// - extracts __heap_base global
|
/// - extracts __heap_base global
|
||||||
/// - computes module max memory size by summation of heap and globals sizes
|
/// - computes module max memory size by summation of heap and globals sizes
|
||||||
/// - sets computed value as max memory page count of a module
|
/// - sets computed value as max memory page count of a module
|
||||||
pub(crate) fn prepare_module(module: &[u8], max_heap_size: u32) -> PrepareResult<Vec<u8>> {
|
pub(crate) fn prepare_module(module: &[u8], max_heap_pages_count: u32) -> PrepareResult<Vec<u8>> {
|
||||||
ModuleBootstrapper::init(module)?
|
ModuleBootstrapper::init(module)?
|
||||||
.set_max_heap_size(max_heap_size)?
|
.set_max_heap_pages_count(max_heap_pages_count)?
|
||||||
.into_wasm()
|
.into_wasm()
|
||||||
}
|
}
|
||||||
|
Loading…
x
Reference in New Issue
Block a user