port SQLite fork to wasm

.gitignore

@@ -0,0 +1,88 @@
+.*.swp
+*.o
+*.log
+dump.rdb
+redis-benchmark
+redis-check-aof
+redis-check-rdb
+redis-check-dump
+redis-cli
+redis-sentinel
+redis-server
+doc-tools
+release
+misc/*
+src/release.h
+appendonly.aof
+SHORT_TERM_TODO
+release.h
+src/transfer.sh
+src/configs
+redis.ds
+src/redis.conf
+src/nodes.conf
+deps/lua/src/lua
+deps/lua/src/luac
+deps/lua/src/liblua.a
+.make-*
+.prerequisites
+*.dSYM
+Makefile.dep
+
+
+### C++ template
+# Compiled Object files
+*.slo
+*.lo
+*.o
+*.obj
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
+# Fortran module files
+*.mod
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.lib
+
+# Executables
+*.exe
+*.out
+*.app
+
+### JetBrains template
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+.idea/*
+cmake-build-debug
+
+## File-based project format:
+*.iws
+
+## Plugin-specific files:
+
+# IntelliJ
+/out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties

Dockerfile

@@ -0,0 +1,13 @@
+FROM ubuntu:19.04
+
+RUN apt-get update \
+ && apt-get install -y ca-certificates \
+                       curl \
+                       git \
+                       make
+
+RUN curl -L https://github.com/CraneStation/wasi-sdk/releases/download/wasi-sdk-5/wasi-sdk-5.0-linux.tar.gz | tar xz --strip-components=1 -C /
+
+VOLUME /code
+WORKDIR /code
+CMD make

Makefile

@@ -0,0 +1,24 @@
+TARGET = sqlite3
+CC = /opt/wasi-sdk/bin/clang
+SYSROOT = /opt/wasi-sdk/share/sysroot
+TARGET_TRIPLE = wasm32-unknown-wasi
+CFLAGS = -nostartfiles -fvisibility=hidden
+LDFLAGS = -Wl,--no-entry,--demangle,--allow-undefined
+EXPORT_FUNCS = --export=allocate,--export=deallocate,--export=invoke
+SQLITE_SRC = src/alter.c src/analyze.c src/attach.c src/auth.c src/backup.c src/bitvec.c src/btmutex.c src/btree.c src/build.c src/callback.c src/complete.c src/ctime.c src/date.c src/dbpage.c src/dbstat.c src/delete.c src/expr.c src/fault.c src/fkey.c src/fts3.c src/fts3_write.c src/fts3_aux.c src/fts3_expr.c src/fts3_hash.c src/fts3_icu.c src/fts3_porter.c src/fts3_snippet.c src/fts3_tokenize_vtab.c src/fts3_tokenizer.c src/fts3_tokenizer1.c src/fts3_unicode.c src/fts3_unicode2.c src/func.c src/global.c src/hash.c src/insert.c src/json1.c src/legacy.c src/loadext.c src/main.c src/malloc.c src/memdb.c src/mem0.c src/mem1.c src/mem2.c src/memjournal.c src/notify.c src/opcodes.c src/os.c src/pager.c src/parse.c src/pcache.c src/pcache1.c src/pragma.c src/prepare.c src/printf.c src/random.c src/resolve.c src/rowset.c src/rtree.c src/select.c src/sqlite3session.c src/status.c src/stmt.c src/table.c src/tokenize.c src/treeview.c src/trigger.c src/update.c src/upsert.c src/userauth.c src/utf.c src/util.c src/vacuum.c src/vdbe.c src/vdbeapi.c src/vdbeaux.c src/vdbeblob.c src/vdbemem.c src/vdbesort.c src/vdbetrace.c src/vtab.c src/wal.c src/walker.c src/where.c src/wherecode.c src/whereexpr.c src/window.c
+WRAPPER_SRC = src/wrapper.c
+SQLITE_FLAGS = -D_HAVE_SQLITE_CONFIG_H -DBUILD_sqlite -DNDEBUG -DSQLITE_THREADSAFE=0 -DHAVE_READLINE=0 -DHAVE_EDITLINE=0 -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION -DSQLITE_ENABLE_STMTVTAB -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_DBSTAT_VTAB -DSQLITE_ENABLE_OFFSET_SQL_FUNC -DSQLITE_ENABLE_DESERIALIZE -DSQLITE_INTROSPECTION_PRAGMAS -DSQLITE_OMIT_POPEN
+SDK = sdk/logger.c
+
+.PHONY: default all clean
+
+default: $(TARGET)
+all: default
+
+$(TARGET): $(SDK) $(SQLITE_SRC) $(WRAPPER_SRC)
+	$(CC) --sysroot=$(SYSROOT) --target=$(TARGET_TRIPLE) -O2 $(SQLITE_FLAGS) $(CFLAGS) $(LDFLAGS) -Wl,$(EXPORT_FUNCS) $^ -o $@.wasm
+
+.PRECIOUS: $(TARGET)
+
+clean:
+	-rm -f $(TARGET).wasm

Readme.md

@@ -0,0 +1,16 @@
+# SQLite
+
+Sqlite fork ported to WebAssembly.
+
+# How to build
+
+This app could be built either with docker 
+
+```bash
+docker-compose up
+```
+
+or by Makefile with [wasi-sdk](https://github.com/CraneStation/wasi-sdk) installed
+```bash
+make
+```

docker-compose.yml

@@ -0,0 +1,7 @@
+version: '3'
+services:
+  sqlite3:
+    build:
+      context: .
+    volumes:
+     - .:/code

sdk/allocator.c

@@ -0,0 +1,13 @@
+#include "allocator.h"
+#include <stdlib.h>
+
+#define UNUSED(x) (void)(x)
+
+void *allocate(size_t size) {
+    return malloc(size);
+}
+
+void deallocate(void *ptr, size_t size) {
+    UNUSED(size);
+    free(ptr);
+}

sdk/allocator.h

@@ -0,0 +1,27 @@
+#ifndef C_SDK_ALLOCATOR_H
+#define C_SDK_ALLOCATOR_H
+
+#include <stddef.h> // for size_t
+
+/**
+ * Allocates a memory region of given size.
+ *
+ * Used by Wasm VM for byte array passing. Should be exported from module.
+ *
+ * @param size a size of needed memory region.
+ * @return a pointer to allocated memory region.
+ */
+void *allocate(size_t size);
+
+/**
+ * Frees a memory region.
+ *
+ * Used by Wasm VM for freeing previous memory allocated by `allocate` function.
+ * Should be exported from module.
+ *
+ * @param ptr the pointer to the previously allocated memory region.
+ * @param size the size of the previously allocated memory region.
+ */
+void deallocate(void *ptr, size_t size);
+
+#endif //C_SDK_ALLOCATOR_H

sdk/logger.c

@@ -0,0 +1,15 @@
+#include "logger.h"
+
+#define __LOGGER_IMPORT(name) \
+    __attribute__((__import_module__("logger"), __import_name__(#name)))
+
+void __write(char ch) __LOGGER_IMPORT(write);
+void __flush() __LOGGER_IMPORT(flush);
+
+void wasm_log(const char *str, int len) {
+    for(int byteId = 0; byteId < len; ++byteId) {
+        __write(str[byteId]);
+    }
+
+    __flush();
+}

sdk/logger.h

@@ -0,0 +1,11 @@
+#ifndef C_SDK_LOGGER_H
+#define C_SDK_LOGGER_H
+
+/**
+ * Writes provided string to Wasm VM logger.
+ *
+ * @param log_message a message that should be logged.
+ */
+void wasm_log(const char *str, int len);
+
+#endif //C_SDK_LOGGER_H

src/config.h

@@ -0,0 +1,132 @@
+/* config.h.  Generated from config.h.in by configure.  */
+/* config.h.in.  Generated from configure.ac by autoheader.  */
+
+/* Define to 1 if you have the <dlfcn.h> header file. */
+#define HAVE_DLFCN_H 1
+
+/* Define to 1 if you have the `fdatasync' function. */
+#define HAVE_FDATASYNC 1
+
+/* Define to 1 if you have the `gmtime_r' function. */
+#define HAVE_GMTIME_R 1
+
+/* Define to 1 if the system has the type `int16_t'. */
+#define HAVE_INT16_T 1
+
+/* Define to 1 if the system has the type `int32_t'. */
+#define HAVE_INT32_T 1
+
+/* Define to 1 if the system has the type `int64_t'. */
+#define HAVE_INT64_T 1
+
+/* Define to 1 if the system has the type `int8_t'. */
+#define HAVE_INT8_T 1
+
+/* Define to 1 if the system has the type `intptr_t'. */
+#define HAVE_INTPTR_T 1
+
+/* Define to 1 if you have the <inttypes.h> header file. */
+#define HAVE_INTTYPES_H 1
+
+/* Define to 1 if you have the `isnan' function. */
+#define HAVE_ISNAN 1
+
+/* Define to 1 if you have the `localtime_r' function. */
+#define HAVE_LOCALTIME_R 1
+
+/* Define to 1 if you have the `localtime_s' function. */
+/* #undef HAVE_LOCALTIME_S */
+
+/* Define to 1 if you have the <malloc.h> header file. */
+#define HAVE_MALLOC_H 1
+
+/* Define to 1 if you have the `malloc_usable_size' function. */
+#define HAVE_MALLOC_USABLE_SIZE 1
+
+/* Define to 1 if you have the <memory.h> header file. */
+#define HAVE_MEMORY_H 1
+
+/* Define to 1 if you have the pread() function. */
+#define HAVE_PREAD 1
+
+/* Define to 1 if you have the pread64() function. */
+#define HAVE_PREAD64 1
+
+/* Define to 1 if you have the pwrite() function. */
+#define HAVE_PWRITE 1
+
+/* Define to 1 if you have the pwrite64() function. */
+#define HAVE_PWRITE64 1
+
+/* Define to 1 if you have the <stdint.h> header file. */
+#define HAVE_STDINT_H 1
+
+/* Define to 1 if you have the <stdlib.h> header file. */
+#define HAVE_STDLIB_H 1
+
+/* Define to 1 if you have the strchrnul() function */
+#define HAVE_STRCHRNUL 1
+
+/* Define to 1 if you have the <strings.h> header file. */
+#define HAVE_STRINGS_H 1
+
+/* Define to 1 if you have the <string.h> header file. */
+#define HAVE_STRING_H 1
+
+/* Define to 1 if you have the <sys/stat.h> header file. */
+#define HAVE_SYS_STAT_H 1
+
+/* Define to 1 if you have the <sys/types.h> header file. */
+#define HAVE_SYS_TYPES_H 1
+
+/* Define to 1 if the system has the type `uint16_t'. */
+#define HAVE_UINT16_T 1
+
+/* Define to 1 if the system has the type `uint32_t'. */
+#define HAVE_UINT32_T 1
+
+/* Define to 1 if the system has the type `uint64_t'. */
+#define HAVE_UINT64_T 1
+
+/* Define to 1 if the system has the type `uint8_t'. */
+#define HAVE_UINT8_T 1
+
+/* Define to 1 if the system has the type `uintptr_t'. */
+#define HAVE_UINTPTR_T 1
+
+/* Define to 1 if you have the <unistd.h> header file. */
+#define HAVE_UNISTD_H 1
+
+/* Define to 1 if you have the `usleep' function. */
+#define HAVE_USLEEP 1
+
+/* Define to 1 if you have the utime() library function. */
+#define HAVE_UTIME 1
+
+/* Define to the sub-directory in which libtool stores uninstalled libraries.
+   */
+#define LT_OBJDIR ".libs/"
+
+/* Define to the address where bug reports for this package should be sent. */
+#define PACKAGE_BUGREPORT ""
+
+/* Define to the full name of this package. */
+#define PACKAGE_NAME "sqlite"
+
+/* Define to the full name and version of this package. */
+#define PACKAGE_STRING "sqlite 3.30.0"
+
+/* Define to the one symbol short name of this package. */
+#define PACKAGE_TARNAME "sqlite"
+
+/* Define to the version of this package. */
+#define PACKAGE_VERSION "3.30.0"
+
+/* Define to 1 if you have the ANSI C header files. */
+#define STDC_HEADERS 1
+
+/* Number of bits in a file offset, on hosts where this is settable. */
+/* #undef _FILE_OFFSET_BITS */
+
+/* Define for large files, on AIX-style hosts. */
+/* #undef _LARGE_FILES */

src/main.c

@@ -237,7 +237,9 @@ int sqlite3_initialize(void){
     }
     if( rc==SQLITE_OK ){
       sqlite3GlobalConfig.isPCacheInit = 1;
+#if __sqlite_unmodified_upstream
       rc = sqlite3OsInit();
+#endif
     }
 #ifdef SQLITE_ENABLE_DESERIALIZE
     if( rc==SQLITE_OK ){
@@ -321,7 +323,9 @@ int sqlite3_shutdown(void){
     void SQLITE_EXTRA_SHUTDOWN(void);
     SQLITE_EXTRA_SHUTDOWN();
 #endif
+#if __sqlite_unmodified_upstream
     sqlite3_os_end();
+#endif
     sqlite3_reset_auto_extension();
     sqlite3GlobalConfig.isInit = 0;
   }

src/memdb.c

@@ -17,6 +17,10 @@
 ** sqlite3_deserialize().
 */
 #include "sqliteInt.h"
+#if __sqlite_unmodified_upstream
+#else
+#include <stdlib.h>
+#endif
 #ifdef SQLITE_ENABLE_DESERIALIZE
 
 /*
@@ -422,7 +426,15 @@ static void memdbDlClose(sqlite3_vfs *pVfs, void *pHandle){
 ** random data.
 */
 static int memdbRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){
+#if __sqlite_unmodified_upstream
   return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut);
+#else
+  zBufOut = malloc(nByte);
+  for(int i = 0; i < nByte; ++i) {
+    zBufOut[i] = rand() % 256;
+  }
+  return SQLITE_OK;
+#endif
 }
 
 /*
@@ -610,14 +622,23 @@ end_deserialize:
 ** Register the new VFS.
 */
 int sqlite3MemdbInit(void){
+#if __sqlite_unmodified_upstream
   sqlite3_vfs *pLower = sqlite3_vfs_find(0);
   int sz = pLower->szOsFile;
   memdb_vfs.pAppData = pLower;
+#else
+  memdb_vfs.pAppData = (void *)0xFFFFFFFF;
+  int sz = sizeof(MemFile);
+#endif
   /* In all known configurations of SQLite, the size of a default
   ** sqlite3_file is greater than the size of a memdb sqlite3_file.
   ** Should that ever change, remove the following NEVER() */
   if( NEVER(sz<sizeof(MemFile)) ) sz = sizeof(MemFile);
   memdb_vfs.szOsFile = sz;
+#if __sqlite_unmodified_upstream // Make memdb the default database
   return sqlite3_vfs_register(&memdb_vfs, 0);
+#else
+  return sqlite3_vfs_register(&memdb_vfs, 1);
+#endif
 }
 #endif /* SQLITE_ENABLE_DESERIALIZE */

src/wrapper.c

@@ -0,0 +1,145 @@
+#include <stdlib.h>
+#include "../sdk/logger.h"
+#include "sqliteInt.h"
+
+sqlite3 *state;
+
+int init() {
+  const int rc = sqlite3_initialize();
+  if(rc != 0) {
+    return rc;
+  }
+
+  return sqlite3_open(":memory:", &state);
+}
+
+int g_isInited = 0;
+
+void* allocate(size_t size) {
+  return malloc(size + 1);
+}
+
+void deallocate(void *ptr, int size) {
+  free(ptr);
+}
+
+char *write_response(char *response, int response_size) {
+  char *result_response = allocate(response_size + 4);
+
+  for(int i = 0; i < 4; ++i) {
+    result_response[i] = (response_size >> 8*i) & 0xFF;
+  }
+
+  memcpy(result_response + 4, response, response_size);
+  return result_response;
+}
+
+typedef struct ShellText ShellText;
+struct ShellText {
+    char *z;
+    int n;
+    int nAlloc;
+};
+
+static void initText(ShellText *p){
+  memset(p, 0, sizeof(*p));
+}
+
+static void freeText(ShellText *p){
+  free(p->z);
+  initText(p);
+}
+
+static int strlen30(const char *z){
+  const char *z2 = z;
+  while( *z2 ){ z2++; }
+  return 0x3fffffff & (int)(z2 - z);
+}
+
+static void appendText(ShellText *p, char const *zAppend, char quote){
+  int len;
+  int i;
+  int nAppend = strlen30(zAppend);
+
+  len = nAppend+p->n+1;
+  if( quote ){
+    len += 2;
+    for(i=0; i<nAppend; i++){
+      if( zAppend[i]==quote ) len++;
+    }
+  }
+
+  if( p->n+len>=p->nAlloc ){
+    p->nAlloc = p->nAlloc*2 + len + 20;
+    p->z = realloc(p->z, p->nAlloc);
+    // TODO: more solid work with OOM
+    if( p->z==0 ) __builtin_unreachable();
+  }
+
+  if( quote ){
+    char *zCsr = p->z+p->n;
+    *zCsr++ = quote;
+    for(i=0; i<nAppend; i++){
+      *zCsr++ = zAppend[i];
+      if( zAppend[i]==quote ) *zCsr++ = quote;
+    }
+    *zCsr++ = quote;
+    p->n = (int)(zCsr - p->z);
+    *zCsr = '\0';
+  }else{
+    memcpy(p->z+p->n, zAppend, nAppend);
+    p->n += nAppend;
+    p->z[p->n] = '\0';
+  }
+}
+
+static int captureOutputCallback(void *pArg, int nArg, char **azArg, char **az){
+  ShellText *p = (ShellText*)pArg;
+  int i;
+  UNUSED_PARAMETER(az);
+  if( azArg==0 ) return 0;
+  if( p->n ) appendText(p, "|", 0);
+  for(i=0; i<nArg; i++){
+    if( i ) appendText(p, ",", 0);
+    if( azArg[i] ) appendText(p, azArg[i], 0);
+  }
+  return 0;
+}
+
+const char *invoke(char *request, int request_size) {
+  if(g_isInited == 0) {
+    // TODO: check the return code
+    init();
+    const char successInitMessage[] = "Sqlite has been initialized";
+    wasm_log(successInitMessage, sizeof(successInitMessage));
+    g_isInited = 1;
+  }
+
+  request[request_size] = 0;
+
+  wasm_log(request, request_size);
+
+  ShellText str;
+  initText(&str);
+  char *errorMessage = 0;
+
+  int rc = sqlite3_exec(state, request, captureOutputCallback, &str, &errorMessage);
+
+  char *response = 0;
+  if(rc || errorMessage) {
+    response = write_response(errorMessage, strlen(errorMessage));
+  }
+  else {
+    if(str.n != 0) {
+      response = write_response(str.z, str.n);
+    } else {
+      // if a request was successfull, sqlite doesn't return anything as the result string
+      const char success_result[] = "OK";
+      response = write_response((char *)success_result, sizeof(success_result));
+    }
+  }
+
+  freeText(&str);
+
+  return response;
+}