commit 9db382cdd496240eb5e4b3fff9bb58985d50d7d7 Author: Alexey Proshutinskiy Date: Thu Jun 24 13:59:55 2021 +0300 initial commit diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..b6e7a1e --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,35 @@ +version: 2.1 + +orbs: + docker: circleci/docker@1.5.0 + +jobs: + Build: + docker: + - image: circleci/rust:latest + resource_class: xlarge + environment: + RUST_BACKTRACE: 1 + steps: + - checkout + - run: | + sudo bash .github/download_marine.sh + - restore_cache: + keys: + - ipfs-adapter00-{{ checksum "Cargo.lock" }} + - run: | + rustup toolchain install nightly-2021-04-24-x86_64-unknown-linux-gnu + rustup default nightly-2021-04-24-x86_64-unknown-linux-gnu + rustup target add wasm32-wasi --toolchain nightly-2021-04-24-x86_64-unknown-linux-gnu + ./build.sh + - save_cache: + paths: + - ~/.cargo + - ~/.rustup + key: ipfs-adapter00-{{ checksum "Cargo.lock" }} + +workflows: + version: 2 + CircleCI: + jobs: + - Build \ No newline at end of file diff --git a/.github/download_marine.sh b/.github/download_marine.sh new file mode 100755 index 0000000..c7609aa --- /dev/null +++ b/.github/download_marine.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -o pipefail -o errexit -o nounset +set -x + +MARINE_RELEASE="https://api.github.com/repos/fluencelabs/marine/releases/latest" +OUT_DIR=/usr/local/bin + +# get metadata about release +curl -s -H "Accept: application/vnd.github.v3+json" $MARINE_RELEASE | + # extract url and name for asset with name "marine" + # also append $OUT_DIR to each name so file is saved to $OUT_DIR + jq -r ".assets | .[] | select(.name == \"marine\") | \"\(.browser_download_url) $OUT_DIR/\(.name)\"" | + # download assets + xargs -n2 bash -c 'curl -L $0 -o $1 && chmod +x $1' diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7149c8e --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ +# Generated by Cargo +# will have compiled files and executables +*/target/ + +# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries +# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html +Cargo.lock + +# These are backup files generated by rustfmt +**/*.rs.bk + +# Added by cargo +.idea + +/artifacts +g \ No newline at end of file diff --git a/Config.toml b/Config.toml new file mode 100644 index 0000000..c5e30c6 --- /dev/null +++ b/Config.toml @@ -0,0 +1,17 @@ +modules_dir = "artifacts/" + +[[module]] + name = "ipfs_effector" + mem_pages_count = 100 + logger_enabled = true + + [module.mounted_binaries] + ipfs = "/usr/local/bin/ipfs" + + [module.wasi] + envs = { "IPFS_ADDR" = "/dns4/relay02.fluence.dev/tcp/15001", "timeout" = "1s" } + +[[module]] + name = "ipfs_pure" + mem_pages_count = 100 + logger_enabled = true diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..f758a91 --- /dev/null +++ b/build.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -o errexit -o nounset -o pipefail + +# This script builds all subprojects and puts all created Wasm modules in one dir +cd effector +cargo update --aggressive +marine build --release + +cd ../pure +cargo update --aggressive +marine build --release + +cd .. +mkdir -p artifacts +rm -f artifacts/*.wasm +cp effector/target/wasm32-wasi/release/ipfs_effector.wasm artifacts/ +cp pure/target/wasm32-wasi/release/ipfs_pure.wasm artifacts/ diff --git a/effector/Cargo.toml b/effector/Cargo.toml new file mode 100644 index 0000000..606cda4 --- /dev/null +++ b/effector/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "ipfs-effector" +version = "0.1.0" +authors = ["Fluence Labs"] +edition = "2018" +publish = false + +[[bin]] +name = "ipfs_effector" +path = "src/main.rs" + +[dependencies] +marine-rs-sdk = { version = "0.6.10", features = ["logger"] } +log = "0.4.14" \ No newline at end of file diff --git a/effector/src/main.rs b/effector/src/main.rs new file mode 100644 index 0000000..f83b23b --- /dev/null +++ b/effector/src/main.rs @@ -0,0 +1,101 @@ +/* + * Copyright 2020 Fluence Labs Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#![allow(improper_ctypes)] + +mod path; + +use crate::path::to_full_path; + +use marine_rs_sdk::marine; +use marine_rs_sdk::module_manifest; +use marine_rs_sdk::MountedBinaryResult; +use marine_rs_sdk::WasmLoggerBuilder; + +const RESULT_FILE_PATH: &str = "/tmp/ipfs_rpc_file"; +const IPFS_ADDR_ENV_NAME: &str = "IPFS_ADDR"; +const TIMEOUT_ENV_NAME: &str = "timeout"; + +module_manifest!(); + +pub fn main() { + WasmLoggerBuilder::new() + .with_log_level(log::LevelFilter::Info) + .build() + .unwrap(); +} + +/// Put file from specified path to IPFS and return its hash. +#[marine] +pub fn put(file_path: String) -> String { + log::info!("put called with file path {}", file_path); + + let file_path = to_full_path(file_path); + + let timeout = std::env::var(TIMEOUT_ENV_NAME).unwrap_or_else(|_| "1s".to_string()); + let cmd = vec![ + String::from("add"), + String::from("--timeout"), + timeout, + String::from("-Q"), + file_path, + ]; + + let ipfs_result = ipfs(cmd); + ipfs_result + .into_std() + .unwrap() + .unwrap_or_else(std::convert::identity) +} + +/// Get file by provided hash from IPFS, saves it to a temporary file and returns a path to it. +#[marine] +pub fn get(hash: String) -> String { + log::info!("get called with hash {}", hash); + + let result_file_path = to_full_path(RESULT_FILE_PATH); + + let timeout = std::env::var(TIMEOUT_ENV_NAME).unwrap_or_else(|_| "1s".to_string()); + let cmd = vec![ + String::from("get"), + String::from("--timeout"), + timeout, + String::from("-o"), + result_file_path, + hash, + ]; + + ipfs(cmd); + RESULT_FILE_PATH.to_string() +} + +#[marine] +pub fn get_address() -> String { + match std::env::var(IPFS_ADDR_ENV_NAME) { + Ok(addr) => addr, + Err(e) => format!( + "getting {} env variable failed with error {:?}", + IPFS_ADDR_ENV_NAME, e + ), + } +} + +#[marine] +#[link(wasm_import_module = "host")] +extern "C" { + /// Execute provided cmd as a parameters of ipfs cli, return result. + pub fn ipfs(cmd: Vec) -> MountedBinaryResult; +} diff --git a/effector/src/path.rs b/effector/src/path.rs new file mode 100644 index 0000000..3fa025d --- /dev/null +++ b/effector/src/path.rs @@ -0,0 +1,52 @@ +/* + * Copyright 2020 Fluence Labs Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +pub(super) fn to_full_path(cmd: S) -> String +where + S: Into, +{ + use std::path::Component; + use std::path::Path; + + let cmd = cmd.into(); + let path = Path::new(&cmd); + + let mut components = path.components(); + let is_absolute = components.next() == Some(Component::RootDir); + + if !is_absolute { + return cmd; + } + + let parent = match components.next() { + Some(Component::Normal(path)) => path.to_str().unwrap(), + _ => return cmd, + }; + + match std::env::var(parent) { + Ok(to_dir) => { + let mut full_path = std::path::PathBuf::from(to_dir); + + // TODO: optimize this + #[allow(clippy::while_let_on_iterator)] + while let Some(component) = components.next() { + full_path.push(component); + } + full_path.to_string_lossy().into_owned() + } + Err(_) => cmd, + } +} diff --git a/pure/Cargo.toml b/pure/Cargo.toml new file mode 100644 index 0000000..a82d393 --- /dev/null +++ b/pure/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "ipfs-pure" +version = "0.1.0" +authors = ["Fluence Labs"] +edition = "2018" +publish = false + +[[bin]] +name = "ipfs_pure" +path = "src/main.rs" + +[dependencies] +marine-rs-sdk = { version = "0.6.10", features = ["logger"] } +log = "0.4.14" \ No newline at end of file diff --git a/pure/src/main.rs b/pure/src/main.rs new file mode 100644 index 0000000..7a28624 --- /dev/null +++ b/pure/src/main.rs @@ -0,0 +1,74 @@ +/* + * Copyright 2020 Fluence Labs Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#![allow(improper_ctypes)] + +use marine_rs_sdk::marine; +use marine_rs_sdk::module_manifest; +use marine_rs_sdk::WasmLoggerBuilder; + +use std::fs; +use std::path::PathBuf; + +const RPC_TMP_FILEPATH: &str = "/tmp/ipfs_rpc_file"; + +module_manifest!(); + +pub fn main() { + WasmLoggerBuilder::new() + .with_log_level(log::LevelFilter::Info) + .build() + .unwrap(); +} + +#[marine] +pub fn invoke() -> String { + "IPFS_RPC wasm example, it allows to:\ninvoke\nput\nget".to_string() +} + +#[marine] +pub fn put(file_content: Vec) -> String { + log::info!("put called with {:?}", file_content); + + let rpc_tmp_filepath = RPC_TMP_FILEPATH.to_string(); + + let r = fs::write(PathBuf::from(rpc_tmp_filepath.clone()), file_content); + if let Err(e) = r { + return format!("file can't be written: {}", e); + } + + ipfs_put(rpc_tmp_filepath) +} + +#[marine] +pub fn get(hash: String) -> Vec { + log::info!("get called with hash: {}", hash); + + let file_path = ipfs_get(hash); + fs::read(file_path).unwrap_or_else(|_| b"error while reading file".to_vec()) +} + +#[marine] +#[link(wasm_import_module = "ipfs_effector")] +extern "C" { + /// Put provided file to ipfs, return ipfs hash of the file. + #[link_name = "put"] + pub fn ipfs_put(file_path: String) -> String; + + /// Get file from ipfs by hash. + #[link_name = "get"] + pub fn ipfs_get(hash: String) -> String; +}