fluencelabs/marine
1
0
Fork 0
mirror of https://github.com/fluencelabs/marine.git synced 2025-03-15 05:50:49 +00:00
Code Issues Projects Releases Wiki Activity

Introduce wasm_pages_to_bytes; rename max_heap_size to max_heap_pages_count (#136)

Browse Source
This commit is contained in:
Mike Voronov 2021-12-15 11:12:08 +03:00 committed by GitHub
parent 5e07fe0876
commit 7b0686ab2e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 52 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
Cargo.lock generated
View File

@ -767,9 +767,9 @@ dependencies = [
[[package]]
name = "fluence-app-service"
version = "0.11.0"
version = "0.11.1"
dependencies = [
"fluence-faas 0.10.0",
"fluence-faas 0.10.1",
"log",
"maplit",
"serde",
@ -807,7 +807,7 @@ dependencies = [
[[package]]
name = "fluence-faas"
version = "0.10.0"
version = "0.10.1"
dependencies = [
"bytesize",
"cmd_lib",
@ -817,8 +817,8 @@ dependencies = [
"marine-module-interface 0.1.6",
"marine-rs-sdk",
"marine-rs-sdk-main",
"marine-runtime 0.8.0",
"marine-utils 0.3.0",
"marine-runtime 0.8.1",
"marine-utils 0.4.0",
"once_cell",
"pretty_assertions",
"safe-transmute",
@ -1706,7 +1706,7 @@ dependencies = [
[[package]]
name = "marine-runtime"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"anyhow",
"boolinator",
@ -1718,7 +1718,7 @@ dependencies = [
"marine-it-parser 0.6.8",
"marine-module-info-parser 0.2.2",
"marine-module-interface 0.1.6",
"marine-utils 0.3.0",
"marine-utils 0.4.0",
"multimap",
"once_cell",
"parity-wasm 0.42.2",
@ -1793,7 +1793,7 @@ checksum = "8dc5838acba84ce4d802d672afd0814fae0ae7098021ae5b06d975e70d09f812"
[[package]]
name = "marine-utils"
version = "0.3.0"
version = "0.4.0"
[[package]]
name = "matches"
@ -1913,7 +1913,7 @@ dependencies = [
"check-latest",
"clap",
"env_logger 0.7.1",
"fluence-app-service 0.11.0",
"fluence-app-service 0.11.1",
"itertools 0.9.0",
"log",
"marine-rs-sdk-main",

2
crates/utils/Cargo.toml
View File

@ -1,7 +1,7 @@
[package]
name = "marine-utils"
description = "Fluence Marine utils crate"
version = "0.3.0"
version = "0.4.0"
authors = ["Fluence Labs"]
license = "Apache-2.0"
edition = "2018"

6
crates/utils/src/wasm_mem_pages_conversion.rs
View File

@ -16,10 +16,14 @@
pub const WASM_PAGE_SIZE: u32 = 65356;
pub fn to_wasm_page_count_ceil(offset: u32) -> u32 {
pub fn bytes_to_wasm_pages_ceil(offset: u32) -> u32 {
match offset {
0 => 0,
// ceiling
n => 1 + (n - 1) / WASM_PAGE_SIZE,
}
}
pub fn wasm_pages_to_bytes(pages_count: u32) -> u64 {
(pages_count as u64) * (WASM_PAGE_SIZE as u64)
}

4
fluence-app-service/Cargo.toml
View File

@ -1,13 +1,13 @@
[package]
name = "fluence-app-service"
description = "Fluence Application Service"
version = "0.11.0"
version = "0.11.1"
authors = ["Fluence Labs"]
license = "Apache-2.0"
edition = "2018"
[dependencies]
fluence-faas = { path = "../fluence-faas", version = "0.10.0" }
fluence-faas = { path = "../fluence-faas", version = "0.10.1" }
maplit = "1.0.2"
log = "0.4.8"

6
fluence-faas/Cargo.toml
View File

@ -1,15 +1,15 @@
[package]
name = "fluence-faas"
description = "Fluence FaaS"
version = "0.10.0"
version = "0.10.1"
authors = ["Fluence Labs"]
license = "Apache-2.0"
edition = "2018"
[dependencies]
marine-runtime = { path = "../runtime", version = "0.8.0" }
marine-runtime = { path = "../runtime", version = "0.8.1" }
marine-module-interface = { path = "../crates/module-interface", version = "0.1.6" }
marine-utils = { path = "../crates/utils", version = "0.3.0" }
marine-utils = { path = "../crates/utils", version = "0.4.0" }
marine-rs-sdk-main = { version = "0.6.15", features = ["logger"] }
marine-rs-sdk = { version = "0.6.15", features = ["logger"] }

8
fluence-faas/src/config/to_marine_config.rs
View File

@ -26,7 +26,7 @@ use crate::host_imports::create_call_parameters_import;
use marine::HostImportDescriptor;
use marine::MModuleConfig;
use marine_rs_sdk::CallParameters;
use marine_utils::to_wasm_page_count_ceil;
use marine_utils::bytes_to_wasm_pages_ceil;
use wasmer_core::import::ImportObject;
use wasmer_core::import::Namespace;
use wasmer_runtime::func;
@ -127,7 +127,7 @@ impl MModuleConfigBuilder {
mem_pages_count: Option<u32>,
max_heap_size: Option<u64>,
) -> FaaSResult<Self> {
let mem_pages_count = match (mem_pages_count, max_heap_size) {
let max_heap_pages_count = match (mem_pages_count, max_heap_size) {
(Some(v), None) => v,
(_, Some(max_heap_size_wanted)) => {
if max_heap_size_wanted > WASM_MAX_HEAP_SIZE {
@ -136,13 +136,13 @@ impl MModuleConfigBuilder {
max_heap_size_allowed: WASM_MAX_HEAP_SIZE,
});
};
to_wasm_page_count_ceil(max_heap_size_wanted as u32)
bytes_to_wasm_pages_ceil(max_heap_size_wanted as u32)
}
// leave the default value
(None, None) => return Ok(self),
};
self.config.max_heap_size = mem_pages_count;
self.config.max_heap_pages_count = max_heap_pages_count;
Ok(self)
}

4
runtime/Cargo.toml
View File

@ -1,7 +1,7 @@
[package]
name = "marine-runtime"
description = "Marine is the Fluence Compute Runtime"
version = "0.8.0"
version = "0.8.1"
authors = ["Fluence Labs"]
license = "Apache-2.0"
edition = "2018"
@ -16,7 +16,7 @@ marine-it-interfaces = { path = "../crates/it-interfaces", version = "0.4.1" }
marine-it-parser = { path = "../crates/it-parser", version = "0.6.8" }
marine-it-generator = { path = "../crates/it-generator", version = "0.5.6" }
marine-module-interface = { path = "../crates/module-interface", version = "0.1.6" }
marine-utils = { path = "../crates/utils", version = "0.3.0" }
marine-utils = { path = "../crates/utils", version = "0.4.0" }
wasmer-runtime = { package = "wasmer-runtime-fl", version = "=0.17.1" }
# dynamicfunc-fat-closures allows using state inside DynamicFunc

16
runtime/src/config.rs
View File

@ -26,6 +26,9 @@ use std::path::PathBuf;
use std::collections::HashMap;
use std::collections::HashSet;
// 65536*1600 ~ 100 Mb (Wasm page size is 64 Kb)
const DEFAULT_HEAP_PAGES_COUNT: u32 = 1600;
pub type HostExportedFunc = Box<dyn Fn(&mut Ctx, Vec<IValue>) -> Option<IValue> + 'static>;
pub struct HostImportDescriptor {
@ -46,7 +49,7 @@ pub struct HostImportDescriptor {
pub struct MModuleConfig {
/// Maximum number of Wasm memory pages that loaded module can use.
/// Each Wasm page is 65536 bytes long.
pub max_heap_size: u32,
pub max_heap_pages_count: u32,
/// Import object that will be used in module instantiation process.
pub raw_imports: ImportObject,
@ -71,8 +74,7 @@ impl Default for MModuleConfig {
fn default() -> Self {
// some reasonable defaults
Self {
// 65536*1600 ~ 100 Mb
max_heap_size: 1600,
max_heap_pages_count: DEFAULT_HEAP_PAGES_COUNT,
raw_imports: ImportObject::new(),
host_imports: HashMap::new(),
wasi_version: WasiVersion::Latest,
@ -85,32 +87,28 @@ impl Default for MModuleConfig {
// TODO: implement debug for MModuleConfig
#[allow(dead_code)]
impl MModuleConfig {
#[allow(dead_code)]
pub fn with_mem_pages_count(mut self, mem_pages_count: u32) -> Self {
self.max_heap_size = mem_pages_count;
self.max_heap_pages_count = mem_pages_count;
self
}
#[allow(dead_code)]
pub fn with_wasi_version(mut self, wasi_version: WasiVersion) -> Self {
self.wasi_version = wasi_version;
self
}
#[allow(dead_code)]
pub fn with_wasi_envs(mut self, envs: HashMap<Vec<u8>, Vec<u8>>) -> Self {
self.wasi_envs = envs;
self
}
#[allow(dead_code)]
pub fn with_wasi_preopened_files(mut self, preopened_files: HashSet<PathBuf>) -> Self {
self.wasi_preopened_files = preopened_files;
self
}
#[allow(dead_code)]
pub fn with_wasi_mapped_dirs(mut self, mapped_dirs: HashMap<String, PathBuf>) -> Self {
self.wasi_mapped_dirs = mapped_dirs;
self

3
runtime/src/engine.rs
View File

@ -73,7 +73,8 @@ impl Marine {
wasm_bytes: &[u8],
config: MModuleConfig,
) -> MResult<()> {
let prepared_wasm_bytes = crate::misc::prepare_module(wasm_bytes, config.max_heap_size)?;
let prepared_wasm_bytes =
crate::misc::prepare_module(wasm_bytes, config.max_heap_pages_count)?;
let module = MModule::new(&name, &prepared_wasm_bytes, config, &self.modules)?;
match self.modules.entry(name) {

6
runtime/src/misc/errors.rs
View File

@ -27,10 +27,10 @@ pub enum PrepareError {
#[error(transparent)]
HeapBaseInvalidOrMissing(#[from] HeapBaseError),
#[error("overflow was happened while summation globals size '{globals_size}' and heap size '{max_heap_size}'")]
#[error("overflow was happened while summation globals size '{globals_pages_count}' and heap size '{max_heap_pages_count}'")]
MemSizesOverflow {
globals_size: u32,
max_heap_size: u32,
globals_pages_count: u32,
max_heap_pages_count: u32,
},
/// Error is encountered while parsing module version.

29
runtime/src/misc/prepare.rs
View File

@ -24,7 +24,7 @@ use super::PrepareResult;
use crate::misc::PrepareError;
use heap_base::get_heap_base;
use marine_utils::to_wasm_page_count_ceil;
use marine_utils::bytes_to_wasm_pages_ceil;
use parity_wasm::builder;
use parity_wasm::elements;
@ -43,20 +43,19 @@ impl<'a> ModuleBootstrapper {
Ok(Self { module })
}
fn set_max_heap_size(self, max_heap_size: u32) -> PrepareResult<Self> {
fn set_max_heap_pages_count(self, max_heap_pages_count: u32) -> PrepareResult<Self> {
use elements::{MemoryType, MemorySection};
let Self { mut module } = self;
let globals_size = get_heap_base(&module)
.map(to_wasm_page_count_ceil)
let globals_pages_count = get_heap_base(&module)
.map(bytes_to_wasm_pages_ceil)
.unwrap_or(DEFAULT_GLOBALS_SIZE);
let max_mem_size =
globals_size
.checked_add(max_heap_size)
.ok_or(PrepareError::MemSizesOverflow {
globals_size,
max_heap_size,
})?;
let max_mem_pages_count = globals_pages_count
.checked_add(max_heap_pages_count)
.ok_or(PrepareError::MemSizesOverflow {
globals_pages_count,
max_heap_pages_count,
})?;
// At now, there is could be only one memory section, so
// it needs just to extract previous initial page count,
@ -68,9 +67,9 @@ impl<'a> ModuleBootstrapper {
},
None => 0,
};
let mem_initial_size = std::cmp::min(mem_initial_size, max_mem_size);
let mem_initial_size = std::cmp::min(mem_initial_size, max_mem_pages_count);
let memory_entry = MemoryType::new(mem_initial_size, Some(max_mem_size));
let memory_entry = MemoryType::new(mem_initial_size, Some(max_mem_pages_count));
let mut default_mem_section = MemorySection::default();
module
@ -94,8 +93,8 @@ impl<'a> ModuleBootstrapper {
/// - extracts __heap_base global
/// - computes module max memory size by summation of heap and globals sizes
/// - sets computed value as max memory page count of a module
pub(crate) fn prepare_module(module: &[u8], max_heap_size: u32) -> PrepareResult<Vec<u8>> {
pub(crate) fn prepare_module(module: &[u8], max_heap_pages_count: u32) -> PrepareResult<Vec<u8>> {
ModuleBootstrapper::init(module)?
.set_max_heap_size(max_heap_size)?
.set_max_heap_pages_count(max_heap_pages_count)?
.into_wasm()
}