load modules in order specified in a config

This commit is contained in:
vms 2020-08-08 00:52:21 +03:00
parent 7254fe7e8c
commit 5f0a91d9ec
6 changed files with 52 additions and 31 deletions

View File

@ -13,6 +13,6 @@ service_base_dir = "/Users/tmp"
envs = ["IPFS_ADDR=/dns4/relay02.fluence.dev/tcp/15001", "timeout=1s"] envs = ["IPFS_ADDR=/dns4/relay02.fluence.dev/tcp/15001", "timeout=1s"]
[[module]] [[module]]
name = "zipfs_rpc.wasm" name = "ipfs_rpc.wasm"
mem_pages_count = 100 mem_pages_count = 100
logger_enabled = true logger_enabled = true

View File

@ -24,6 +24,9 @@ pub enum FaaSError {
/// An error related to config parsing. /// An error related to config parsing.
ConfigParseError(String), ConfigParseError(String),
/// An error occurred at the instantiation step.
InstantiationError(String),
/// Various errors related to file i/o. /// Various errors related to file i/o.
IOError(String), IOError(String),
@ -37,6 +40,7 @@ impl std::fmt::Display for FaaSError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
match self { match self {
FaaSError::ConfigParseError(err_msg) => write!(f, "{}", err_msg), FaaSError::ConfigParseError(err_msg) => write!(f, "{}", err_msg),
FaaSError::InstantiationError(err_msg) => write!(f, "{}", err_msg),
FaaSError::IOError(err_msg) => write!(f, "{}", err_msg), FaaSError::IOError(err_msg) => write!(f, "{}", err_msg),
FaaSError::EngineError(err) => write!(f, "{}", err), FaaSError::EngineError(err) => write!(f, "{}", err),
} }

View File

@ -25,6 +25,7 @@ use fce::FCE;
use std::convert::TryInto; use std::convert::TryInto;
use std::collections::HashSet; use std::collections::HashSet;
use std::collections::HashMap;
use std::fs; use std::fs;
use std::path::PathBuf; use std::path::PathBuf;
@ -89,28 +90,38 @@ impl FluenceFaaS {
FaaSError: From<C::Error>, FaaSError: From<C::Error>,
{ {
let config = config.try_into()?; let config = config.try_into()?;
let modules = config.modules_dir.as_ref().map_or(Ok(vec![]), |dir| { let modules = config
Self::load_modules(dir, ModulesLoadStrategy::All) .modules_dir
})?; .as_ref()
.map_or(Ok(HashMap::new()), |dir| {
Self::load_modules(dir, ModulesLoadStrategy::All)
})?;
Self::with_modules::<_, ModulesConfig>(modules, config) Self::with_modules::<ModulesConfig>(modules, config)
} }
/// Creates FaaS with given modules. /// Creates FaaS with given modules.
pub fn with_modules<I, C>(modules: I, config: C) -> Result<Self> pub fn with_modules<C>(mut modules: HashMap<String, Vec<u8>>, config: C) -> Result<Self>
where where
I: IntoIterator<Item = (String, Vec<u8>)>,
C: TryInto<ModulesConfig>, C: TryInto<ModulesConfig>,
FaaSError: From<C::Error>, FaaSError: From<C::Error>,
{ {
let mut fce = FCE::new(); let mut fce = FCE::new();
let mut config = config.try_into()?; let config = config.try_into()?;
for (module_name, module_config) in config.modules_config {
let module_bytes = modules.remove(&module_name).ok_or_else(|| {
FaaSError::InstantiationError(format!(
"module with name {} is specified in config, but not found in provided modules",
module_name
))
})?;
let fce_module_config = crate::misc::make_fce_config(Some(module_config))?;
fce.load_module(module_name, &module_bytes, fce_module_config)?;
}
for (name, bytes) in modules { for (name, bytes) in modules {
let module_config = match config.modules_config.remove(&name) { let module_config = config.default_modules_config.clone();
module_config @ Some(_) => module_config,
None => config.default_modules_config.clone(),
};
let fce_module_config = crate::misc::make_fce_config(module_config)?; let fce_module_config = crate::misc::make_fce_config(module_config)?;
fce.load_module(name.clone(), &bytes, fce_module_config)?; fce.load_module(name.clone(), &bytes, fce_module_config)?;
} }
@ -125,29 +136,32 @@ impl FluenceFaaS {
FaaSError: From<C::Error>, FaaSError: From<C::Error>,
{ {
let config = config.try_into()?; let config = config.try_into()?;
let modules = config.modules_dir.as_ref().map_or(Ok(vec![]), |dir| { let modules = config
Self::load_modules(dir, ModulesLoadStrategy::Named(names)) .modules_dir
})?; .as_ref()
.map_or(Ok(HashMap::new()), |dir| {
Self::load_modules(dir, ModulesLoadStrategy::Named(names))
})?;
Self::with_modules::<_, ModulesConfig>(modules, config) Self::with_modules::<ModulesConfig>(modules, config)
} }
/// Loads modules from a directory at a given path. Non-recursive, ignores subdirectories. /// Loads modules from a directory at a given path. Non-recursive, ignores subdirectories.
fn load_modules( fn load_modules(
modules_dir: &str, modules_dir: &str,
modules: ModulesLoadStrategy<'_>, modules: ModulesLoadStrategy<'_>,
) -> Result<Vec<(String, Vec<u8>)>> { ) -> Result<HashMap<String, Vec<u8>>> {
use FaaSError::IOError; use FaaSError::IOError;
let mut dir_entries = let mut dir_entries =
fs::read_dir(modules_dir).map_err(|e| IOError(format!("{}: {}", modules_dir, e)))?; fs::read_dir(modules_dir).map_err(|e| IOError(format!("{}: {}", modules_dir, e)))?;
let loaded = dir_entries.try_fold(vec![], |mut vec, entry| { let loaded = dir_entries.try_fold(HashMap::new(), |mut hash_map, entry| {
let entry = entry?; let entry = entry?;
let path = entry.path(); let path = entry.path();
// Skip directories // Skip directories
if path.is_dir() { if path.is_dir() {
return Ok(vec); return Ok(hash_map);
} }
let module_name = path let module_name = path
@ -159,10 +173,14 @@ impl FluenceFaaS {
if modules.should_load(&module_name) { if modules.should_load(&module_name) {
let module_bytes = fs::read(path)?; let module_bytes = fs::read(path)?;
vec.push((module_name, module_bytes)); if hash_map.insert(module_name, module_bytes).is_some() {
return Err(FaaSError::ConfigParseError(String::from(
"config contains modules with the same name",
)));
}
} }
Result::Ok(vec) Ok(hash_map)
})?; })?;
if modules.required_modules_len() > loaded.len() { if modules.required_modules_len() > loaded.len() {

View File

@ -17,9 +17,9 @@
use crate::FaaSError; use crate::FaaSError;
use crate::Result; use crate::Result;
use serde_derive::{Serialize, Deserialize}; use serde_derive::Serialize;
use serde_derive::Deserialize;
use std::collections::HashMap;
use std::convert::TryInto; use std::convert::TryInto;
/* /*
@ -39,8 +39,7 @@ service_base_dir = "/Users/user/tmp"
[module.wasi] [module.wasi]
envs = [] envs = []
preopened_files = ["service_id"] preopened_files = ["/Users/user/tmp"]
# it has to be full path from the right side
mapped_dirs = ["tmp" = "/Users/user/tmp"] mapped_dirs = ["tmp" = "/Users/user/tmp"]
[default] [default]
@ -53,7 +52,7 @@ service_base_dir = "/Users/user/tmp"
[default.wasi] [default.wasi]
envs = [] envs = []
preopened_files = ["service_id"] preopened_files = ["/Users/user/tmp"]
mapped_dirs = ["tmp" = "/Users/user/tmp"] mapped_dirs = ["tmp" = "/Users/user/tmp"]
*/ */
@ -127,7 +126,7 @@ pub struct ModulesConfig {
pub modules_dir: Option<String>, pub modules_dir: Option<String>,
/// Settings for a module with particular name. /// Settings for a module with particular name.
pub modules_config: HashMap<String, ModuleConfig>, pub modules_config: Vec<(String, ModuleConfig)>,
/// Settings for a module that name's not been found in modules_config. /// Settings for a module that name's not been found in modules_config.
pub default_modules_config: Option<ModuleConfig>, pub default_modules_config: Option<ModuleConfig>,
@ -222,7 +221,7 @@ fn from_raw_modules_config(config: RawModulesConfig) -> Result<ModulesConfig> {
.module .module
.into_iter() .into_iter()
.map(from_raw_module_config) .map(from_raw_module_config)
.collect::<Result<HashMap<_, _>>>()?; .collect::<Result<Vec<_>>>()?;
let default_modules_config = config let default_modules_config = config
.default .default

View File

@ -121,15 +121,15 @@ fn main() -> Result<(), anyhow::Error> {
} }
Some("interface") => { Some("interface") => {
let interface = app_service.get_interface(); let interface = app_service.get_interface();
println!("application service interface: {}", interface); println!("application service interface:\n{}", interface);
} }
Some("h") | Some("help") | None => { Some("h") | Some("help") | None => {
println!( println!(
"Enter:\n\ "Enter:\n\
new [config_path] - to create a new AppService (old will be removed) new [config_path] - to create a new AppService (current will be removed)\n\
load <module_name> <module_path> - to load a new Wasm module into App service\n\ load <module_name> <module_path> - to load a new Wasm module into App service\n\
unload <module_name> - to unload Wasm module from AppService\n\ unload <module_name> - to unload Wasm module from AppService\n\
call <module_name> <func_name> [args] - to call function with given name on module with given module_name\n\ call <module_name> <func_name> [args] - to call function with func_name of module with module_name\n\
interface - to print public interface of current AppService\n\ interface - to print public interface of current AppService\n\
h/help - to print this message\n\ h/help - to print this message\n\
e/exit/q/quit - to exit" e/exit/q/quit - to exit"