Saving crude state after mem-leak was resolved

Config.toml

@@ -6,8 +6,8 @@ modules_dir = "artifacts/"
     logger_enabled = false
 
     [module.wasi]
-    preopened_files = ["/tmp"]
+    preopened_files = ["./tmp"]
-    mapped_dirs = { "tmp" = "/tmp" }
+    mapped_dirs = { "tmp" = "./tmp" }
 
 [[module]]
     name = "test"
@@ -15,5 +15,5 @@ modules_dir = "artifacts/"
     logger_enabled = false
 
     [module.wasi]
-    preopened_files = ["/tmp"]
+    preopened_files = ["./tmp"]
-    mapped_dirs = { "tmp" = "/tmp" }
+    mapped_dirs = { "tmp" = "./tmp" }

build.sh

@@ -11,8 +11,8 @@ cd "$(dirname "$0")"
 marine build --release --bin test
 
 # copy .wasm to artifacts
-rm -f artifacts/*
+#rm -f artifacts/*
-mkdir -p artifacts
+#mkdir -p artifacts
 cp target/wasm32-wasi/release/test.wasm artifacts/
 
 # download SQLite 3 to use in tests

leak_launcher/main.rs

@@ -1,6 +1,6 @@
 use fluence_app_service::AppService;
-use fluence_app_service::TomlMarineConfig;
 use fluence_app_service::AppServiceConfig;
+use fluence_app_service::TomlMarineConfig;
 use serde_json::json;
 use std::convert::TryInto;
 
@@ -8,7 +8,7 @@ fn main() {
     let config = TomlMarineConfig::load("Config_spell.toml").unwrap();
     let config = AppServiceConfig {
         service_base_dir: std::path::PathBuf::new(),
-        marine_config: config.try_into().unwrap()
+        marine_config: config.try_into().unwrap(),
     };
 
     let service_name = "spell_service";
@@ -17,14 +17,16 @@ fn main() {
     let config = TomlMarineConfig::load("Config_test.toml").unwrap();
     let config = AppServiceConfig {
         service_base_dir: std::path::PathBuf::new(),
-        marine_config: config.try_into().unwrap()
+        marine_config: config.try_into().unwrap(),
     };
 
     let service_name = "test_service";
     let mut test_service = AppService::new(config, service_name, <_>::default()).unwrap();
 
     let db_path = "./tmp/db.sqlite";
-    test_service.call("create_4", json!(db_path), <_>::default()).unwrap();
+    test_service
+        .call("create_4", json!(db_path), <_>::default())
+        .unwrap();
     //test_service.call("set_limit", json!(1*1024*1024), <_>::default()).unwrap();
 
     /*
@@ -35,48 +37,97 @@ fn main() {
         let result = test_service.call("list_push_string_1", json!(["asdasd", "sadsad"]), <_>::default()).unwrap();
     }
      */
-    for i in 0..500 {
+    for i in 0..10001 {
-        if i % 50 == 0 {
+        if i % 10000 == 0 {
-            println!("test insert_5 {}:\n{}", i, test_service.module_memory_stats());
+            println!(
+                "test insert_6 {}:\n{}",
+                i,
+                test_service.module_memory_stats()
+            );
         }
-        let result = test_service.call("insert_5", json!(["asdasd", "sadsad"]), <_>::default()).unwrap();
+        let result = test_service
+            .call("insert_6", json!(["asdasd", "sadsad", i]), <_>::default())
+            .unwrap();
     }
 
-    for i in 0..500 {
+    // for i in 0..500 {
-        if i % 50 == 0 {
+    //     if i % 50 == 0 {
-            println!("test list_push_string_6 {}:\n{}", i, test_service.module_memory_stats());
+    //         println!(
+    //             "test list_push_string_6 {}:\n{}",
+    //             i,
+    //             test_service.module_memory_stats()
+    //         );
+    //     }
+    //     let result = test_service
+    //         .call(
+    //             "list_push_string_6",
+    //             json!(["asdasd", "sadsad"]),
+    //             <_>::default(),
+    //         )
+    //         .unwrap();
+    // }
+
+    // for i in 0..500 {
+    //     if i % 50 == 0 {
+    //         println!(
+    //             "test list_push_string_5 {}:\n{}",
+    //             i,
+    //             test_service.module_memory_stats()
+    //         );
+    //     }
+    //     let unique_key = create_unique_key(500 + i);
+    //     let result = test_service
+    //         .call(
+    //             "list_push_string_5",
+    //             json!([unique_key, "sadsad"]),
+    //             <_>::default(),
+    //         )
+    //         .unwrap();
+    // }
+
+    // for i in 0..500 {
+    //     if i % 50 == 0 {
+    //         println!(
+    //             "get_all_errors {}:\n{}",
+    //             i,
+    //             spell_service.module_memory_stats()
+    //         );
+    //     }
+    //     let result = spell_service
+    //         .call("insert_2", json!([]), <_>::default())
+    //         .unwrap();
+    // }
+
+    // for i in 0..500 {
+    //     if i % 50 == 0 {
+    //         println!("select_1 {}:\n{}", i, spell_service.module_memory_stats());
+    //     }
+    //     let result = spell_service
+    //         .call("select_1", json!([]), <_>::default())
+    //         .unwrap();
+    // }
+
+    for i in 0..50001 as u32 {
+        if i % 10000 == 0 {
+            println!("select_1 {}:\n{}", i, test_service.module_memory_stats());
         }
-        let result = test_service.call("list_push_string_6", json!(["asdasd", "sadsad"]), <_>::default()).unwrap();
+        let result = test_service
+            .call("select_1", json!([i]), <_>::default())
+            .unwrap();
     }
 
-    for i in 0..500 {
+    // for i in 0..10001 {
-        if i % 50 == 0 {
+    //     if i % 500 == 0 {
-            println!("test list_push_string_5 {}:\n{}", i, test_service.module_memory_stats());
+    //         println!(
-        }
+    //             "test insert_4 {}:\n{}",
-        let unique_key = create_unique_key(500 + i);
+    //             i,
-        let result = test_service.call("list_push_string_5", json!([unique_key, "sadsad"]), <_>::default()).unwrap();
+    //             test_service.module_memory_stats()
-    }
+    //         );
-
+    //     }
-    for i in 0..500 {
+    //     let result = test_service
-        if i % 50 == 0 {
+    //         .call("insert_4", json!(["asdasd", "sadsad"]), <_>::default())
-            println!("get_all_errors {}:\n{}", i, spell_service.module_memory_stats());
+    //         .unwrap();
-        }
+    // }
-        let result = spell_service.call("insert_2", json!([]), <_>::default()).unwrap();
-    }
-
-    for i in 0..500 {
-        if i % 50 == 0 {
-            println!("select_1 {}:\n{}", i, spell_service.module_memory_stats());
-        }
-        let result = spell_service.call("select_1", json!([]), <_>::default()).unwrap();
-    }
-
-    for i in 0..500 {
-        if i % 50 == 0 {
-            println!("select_5 {}:\n{}", i, spell_service.module_memory_stats());
-        }
-        let result = spell_service.call("select_5", json!([]), <_>::default()).unwrap();
-    }
 }
 
 fn create_unique_key(index: i32) -> String {

leak_test/main.rs

@@ -5,10 +5,12 @@ use marine_sqlite_connector::State;
 
 mod schema;
 
-use serde::Serialize;
-use serde::Deserialize;
-use schema::db;
 use fstrings::f;
+use schema::db;
+use schema::get_storage;
+use schema::wrapped_try;
+use serde::Deserialize;
+use serde::Serialize;
 
 fn main() {}
 
@@ -20,7 +22,6 @@ pub struct UnitValue {
     pub error: String,
 }
 
-
 #[marine]
 pub fn create_4(path: String) {
     schema::create();
@@ -147,7 +148,7 @@ fn insert_3() {
 }
 
 #[marine]
-fn insert_4(key: &str, value: String) {
+fn insert_4(key: &str, value: String, i: u32) {
     let conn = db();
 
     let mut statement = conn
@@ -156,17 +157,52 @@ fn insert_4(key: &str, value: String) {
     statement.bind(1, key).expect("bind 1");
     statement.bind(2, value.as_str()).expect("bind 2");
     statement.next().expect("next");
+    // if i % 10000 == 0 {
+    //     println!(
+    //         "sqlite3 memory_used - {:}",
+    //         marine_sqlite_connector::status(0, 0)
+    //     );
+    //     println!(
+    //         "sqlite3 pagecache_used - {:}",
+    //         marine_sqlite_connector::status(1, 0)
+    //     );
+    // }
 }
 
 #[marine]
 fn insert_5(key: &str, value: String) {
-    let mut statement = db().prepare("INSERT OR REPLACE INTO kv (key, string) VALUES (?, ?)")
+    let mut statement = db()
+        .prepare("INSERT OR REPLACE INTO kv (key, string) VALUES (?, ?)")
         .expect("prep rand 0..3");
     statement.bind(1, key).expect("bind 1");
     statement.bind(2, value.as_str()).expect("bind 2");
     statement.next().expect("next");
 }
 
+#[marine]
+fn insert_6(key: &str, value: String, i: u32) {
+    wrapped_try(|| {
+        let storage = get_storage().unwrap();
+        let mut statement = storage
+            .connection
+            .prepare("INSERT OR REPLACE INTO kv (key, string) VALUES (?, ?)")
+            .expect("prep rand 0..3");
+        statement.bind(1, key).expect("bind 1");
+        statement.bind(2, value.as_str()).expect("bind 2");
+        statement.next().expect("next");
+    });
+    // if i % 10000 == 0 {
+    //     println!(
+    //         "sqlite3 memory_used - {:}",
+    //         marine_sqlite_connector::status(0, 0)
+    //     );
+    //     println!(
+    //         "sqlite3 pagecache_used - {:}",
+    //         marine_sqlite_connector::status(1, 0)
+    //     );
+    // }
+}
+
 /*
 #[marine]
 pub fn list_push_string_5(key: &str, value: String) {
@@ -183,8 +219,8 @@ pub fn list_push_string_5(key: &str, value: String) {
  */
 
 #[marine]
-fn select_1(path: String) {
+fn select_1(i: u32) {
-    let conn = marine_sqlite_connector::open(path).expect("Open database connection");
+    let conn = marine_sqlite_connector::open("./tmp/db.sqlite").expect("Open database connection");
 
     let mut statement = conn
         .prepare("SELECT string FROM kv WHERE key = ?")
@@ -194,11 +230,21 @@ fn select_1(path: String) {
     if let State::Row = statement.next().expect("4..5 next") {
         statement.read::<String>(0).expect("4..5 read");
     }
+    // if i % 10000 == 0 {
+    //     println!(
+    //         "sqlite3 memory_used - {:}",
+    //         marine_sqlite_connector::status(0, 0)
+    //     );
+    //     println!(
+    //         "sqlite3 pagecache_used - {:}",
+    //         marine_sqlite_connector::status(1, 0)
+    //     );
+    // }
 }
 
 #[marine]
-fn select_5(path: String) {
+fn select_5() {
-    let conn = marine_sqlite_connector::open(path).expect("Open database connection");
+    let conn = marine_sqlite_connector::open("./tmp/db.sqlite").expect("Open database connection");
 }
 
 #[marine]
@@ -230,7 +276,8 @@ pub fn list_push_string_1(key: &str, value: String) -> UnitValue {
 
 #[marine]
 pub fn list_push_string_2(key: &str, value: String) -> UnitValue {
-        let mut statement = db().prepare(
+    let mut statement = db()
+        .prepare(
             r#"
                 INSERT INTO kv (key, string, list_index)
                     VALUES (
@@ -239,7 +286,8 @@ pub fn list_push_string_2(key: &str, value: String) -> UnitValue {
                         42
                     )
             "#,
-        ).unwrap();
+        )
+        .unwrap();
     statement.bind(1, key).unwrap();
     statement.bind(2, value.as_str()).unwrap();
     statement.next().unwrap();
@@ -249,7 +297,8 @@ pub fn list_push_string_2(key: &str, value: String) -> UnitValue {
 
 #[marine]
 pub fn list_push_string_3(key: &str, value: String) {
-    let mut statement = db().prepare(
+    let mut statement = db()
+        .prepare(
             r#"
                 INSERT INTO kv (key, string, list_index)
                     VALUES (
@@ -258,7 +307,8 @@ pub fn list_push_string_3(key: &str, value: String) {
                         42
                     )
             "#,
-    ).unwrap();
+        )
+        .unwrap();
     statement.bind(1, key).unwrap();
     statement.bind(2, value.as_str()).unwrap();
     statement.next().unwrap();
@@ -266,7 +316,8 @@ pub fn list_push_string_3(key: &str, value: String) {
 
 #[marine]
 pub fn list_push_string_4(key: &str, value: String) {
-    let mut statement = db().prepare(
+    let mut statement = db()
+        .prepare(
             r#"
                 INSERT INTO kv (key, string)
                     VALUES (
@@ -274,7 +325,8 @@ pub fn list_push_string_4(key: &str, value: String) {
                         ?
                     )
             "#,
-    ).unwrap();
+        )
+        .unwrap();
     statement.bind(1, key).unwrap();
     statement.bind(2, value.as_str()).unwrap();
     statement.next().unwrap();
@@ -282,12 +334,14 @@ pub fn list_push_string_4(key: &str, value: String) {
 
 #[marine]
 pub fn list_push_string_5(key: &str, value: String) {
-    let mut statement = db().prepare(
+    let mut statement = db()
+        .prepare(
             r#"
                 INSERT INTO kv (key, string)
                     VALUES (?, ?)
             "#,
-    ).unwrap();
+        )
+        .unwrap();
     statement.bind(1, key).unwrap();
     statement.bind(2, value.as_str()).unwrap();
     statement.next().unwrap();
@@ -295,12 +349,14 @@ pub fn list_push_string_5(key: &str, value: String) {
 
 #[marine]
 pub fn list_push_string_6(key: &str, value: String) {
-    let mut statement = db().prepare(
+    let mut statement = db()
+        .prepare(
             r#"
                 INSERT OR REPLACE INTO kv (key, string)
                     VALUES (?,?)
             "#,
-    ).unwrap();
+        )
+        .unwrap();
     statement.bind(1, key).unwrap();
     statement.bind(2, value.as_str()).unwrap();
     statement.next().unwrap();
@@ -363,4 +419,3 @@ pub enum SpellError {
 pub fn format_error(e: impl std::fmt::Debug) -> String {
     format!("{:?}", e)
 }
-

leak_test/schema.rs

@@ -1,10 +1,32 @@
 use fstrings::f;
 use fstrings::format_args_f;
 use marine_sqlite_connector::Connection;
+use marine_sqlite_connector::Result as SQLiteResult;
 
 pub const DEFAULT_MAX_ERR_PARTICLES: usize = 50;
 pub const DB_FILE: &'static str = "/tmp/db-5.sqlite";
 
+#[derive(Debug)]
+pub struct RegistryResult {
+    pub success: bool,
+    pub error: String,
+}
+
+pub fn wrapped_try<F, T>(func: F) -> T
+where
+    F: FnOnce() -> T,
+{
+    func()
+}
+
+pub struct Storage {
+    pub(crate) connection: Connection,
+}
+
+pub fn get_storage() -> SQLiteResult<Storage> {
+    marine_sqlite_connector::open("./tmp/db.sqlite").map(|c| Storage { connection: c })
+}
+
 pub fn db() -> Connection {
     // use rand::prelude::*;
     //
@@ -17,8 +39,7 @@ pub fn db() -> Connection {
 }
 
 pub fn create() {
-    db().execute(
+    db().execute(f!(r#"
-        f!(r#"
             CREATE TABLE IF NOT EXISTS kv (
                 key TEXT NOT NULL,
                 string TEXT,
@@ -26,7 +47,6 @@ pub fn create() {
 
                 PRIMARY KEY(key, list_index)
             );
-            "#),
+            "#))
-    )
         .expect("init sqlite db");
 }

src/it_generator.rs

@@ -222,3 +222,9 @@ pub fn sqlite3_soft_heap_limit64(size: i64) -> i64 {
 pub fn sqlite3_hard_heap_limit64(size: i64) -> i64 {
     <_>::default()
 }
+
+// sqlite3_int64 sqlite3_hard_heap_limit64(sqlite3_int64 N)
+// #[marine]
+// pub fn sqlite3_status(op: i32, resetFlag: i32) -> i32 {
+//     <_>::default()
+// }

src/lib.rs

@@ -333,3 +333,7 @@ pub fn soft_soft_heap_limit64(limit: i64) -> i64 {
 pub fn set_hard_heap_limit64(limit: i64) -> i64 {
     unsafe { ffi::sqlite3_hard_heap_limit64(limit) }
 }
+
+// pub fn status(op: i32, reset_flag: i32) -> i32 {
+//     unsafe { ffi::sqlite3_status(op, reset_flag) }
+// }

src/sqlite3_connector/import_functions.rs

@@ -141,4 +141,6 @@ extern "C" {
 
     // sqlite3_int64 sqlite3_hard_heap_limit64(sqlite3_int64 N)
     pub fn sqlite3_hard_heap_limit64(size: i64) -> i64;
+
+    // pub fn sqlite3_status(op: i32, reset_flag: i32) -> i32;
 }