ci: Add Aquavm E2E (#324)

.cargo/config.toml

@@ -9,3 +9,6 @@ retry = 50                  # network retries
 rustflags = [
     "-C", "link-args=-zstack-size=52428800",
 ]
+
+[registries]
+fluence = { index = "git://crates.fluence.dev/index" }

.github/download_marine.sh

@@ -6,7 +6,7 @@ MARINE_RELEASE="https://api.github.com/repos/fluencelabs/marine/releases/latest"
 OUT_DIR=/usr/local/bin
 
 # get metadata about release
-curl -s -H "Accept: application/vnd.github.v3+json" $MARINE_RELEASE |
+curl -sS -H "Accept: application/vnd.github.v3+json" $MARINE_RELEASE |
     # extract url and name for asset with name "marine"
     # also append $OUT_DIR to each name so file is saved to $OUT_DIR
     jq -r ".assets | .[] | select(.name == \"marine\") | \"\(.browser_download_url) $OUT_DIR/\(.name)\"" |

.github/workflows/aquavm.yml

@@ -9,7 +9,6 @@ concurrency:
 
 env:
   RUST_TEST_THREADS: 1
-  CARGO_HOME: "${{ github.workspace }}/cargo"
 
 jobs:
   aquavm:

.github/workflows/e2e.yml

@@ -0,0 +1,65 @@
+name: "e2e"
+
+on:
+  pull_request:
+    paths-ignore:
+      - "!**.md"
+
+concurrency:
+  group: "${{ github.workflow }}-${{ github.ref }}"
+  cancel-in-progress: true
+
+jobs:
+  snapshot:
+    name: "aquavm"
+    uses: ./.github/workflows/snapshot.yml
+
+  rust-peer:
+    name: "rust-peer"
+    needs:
+      - snapshot
+
+    uses: fluencelabs/rust-peer/.github/workflows/snapshot.yml@master
+    with:
+      image-name: "docker.fluence.dev/aquavm"
+      avm-server-version: '{ git = "https://github.com/fluencelabs/aquavm", branch = "${{ github.head_ref }}" }'
+      air-interpreter-wasm-version: '{ version = "${{ needs.snapshot.outputs.air-interpreter-wasm-version }}", registry = "fluence" }'
+
+  fluence-js-tests:
+    name: "fluence-js"
+    needs:
+      - snapshot
+
+    uses: fluencelabs/fluence-js/.github/workflows/tests.yml@master
+    with:
+      avm-version: ${{ needs.snapshot.outputs.avm-version }}
+
+  fluence-js-snapshot:
+    name: "fluence-js"
+    needs:
+      - snapshot
+
+    uses: fluencelabs/fluence-js/.github/workflows/snapshot.yml@master
+    with:
+      avm-version: ${{ needs.snapshot.outputs.avm-version }}
+
+  aqua-snapshot:
+    name: "aqua"
+    needs:
+      - fluence-js-snapshot
+
+    uses: fluencelabs/aqua/.github/workflows/snapshot.yml@main
+    with:
+      fluence-js-version: ${{ needs.fluence-js-snapshot.outputs.fluence-js-version }}
+
+  aqua-playground:
+    needs:
+      - fluence-js-snapshot
+      - aqua-snapshot
+      - rust-peer
+
+    uses: fluencelabs/aqua-playground/.github/workflows/tests.yml@master
+    with:
+      fluence-js-version: ${{ needs.fluence-js-snapshot.outputs.fluence-js-version }}
+      aqua-version: ${{ needs.aqua-snapshot.outputs.aqua-version }}
+      rust-peer-image: ${{ needs.rust-peer.outputs.rust-peer-image }}

.github/workflows/publish_interpreter_dev.yml

@@ -1,158 +0,0 @@
-# run locally like this:
-# act -b -P ubuntu-latest=nektos/act-environments-ubuntu:18.04 -j publish-interpreter-branch -s "NPM_TOKEN=uuid-uuid-uuid-uuid"
-
-name: "publish-interpreter-branch"
-
-on:
-  push:
-    branches-ignore:
-      - "master"
-
-jobs:
-  npm-publish-dev:
-    name: "Publish AIR interpreter .wasm to NPM & crates.io"
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Extract branch name
-        if: github.event_name != 'pull_request'
-        run: echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV
-        id: extract_branch
-
-      - name: Extract branch name
-        if: github.event_name == 'pull_request'
-        run: echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV
-
-      - name: Checkout repository
-        uses: actions/checkout@v2
-
-      - name: Setup rust toolchain
-        uses: actions-rust-lang/setup-rust-toolchain@v1
-
-      - run: cargo update --aggressive
-
-      - name: Install jq & sponge
-        run: sudo apt-get update && sudo apt-get --yes --force-yes install jq moreutils
-
-      - name: Install cargo-show & toml-cli
-        run: cargo install cargo-show toml-cli
-
-      - name: Get versions from npm & crates.io, and take the highest one
-        run: |
-          set -x
-          # install semver and add it to PATH
-          yarn global add semver
-          PATH="$(yarn global bin):$PATH"
-
-          INTERPRETER_CARGO_TOML="air-interpreter/Cargo.toml"
-          CARGO_TOML="crates/interpreter-wasm/Cargo.toml"
-          PACKAGE_JSON="avm/client/package.json"
-
-          # sanitize branch name so it can be used as a semver suffix (replace [^0-9a-zA-Z-] with hyphen)
-          SANITIZED_BRANCH="$(echo -n "${{ env.BRANCH_NAME }}" | tr -C '[:alnum:]-' -)"
-          # JQ Version regex pattern
-          PAT="\\\\d+.\\\\d+.\\\\d+-$SANITIZED_BRANCH.\\\\d+"
-
-          # get package name from Cargo.toml
-          PKG_NAME="$(toml get "$CARGO_TOML" package.name | tr -d \")"
-
-          # get package name from package.json
-          JS_PKG_NAME="$(cat "$PACKAGE_JSON" | jq -r .name)"
-
-          ### NPM
-          # take all versions from npm and replace single quotes with double quotes
-          NPM_VERSIONS=$(yarn info --silent "$JS_PKG_NAME" versions 2>/dev/null | tr \' \")
-          # take only versions that contain branch name
-          NPM_VERSIONS_FILTERED=$(echo $NPM_VERSIONS | jq -r ".[] | select(test(\"$PAT\"))")
-          # flatten into a single line
-          NPM_VERSIONS_FLATTENED=$(echo $NPM_VERSIONS_FILTERED | awk '{print}' ORS=' ')
-          # sort versions according to semver, take highest (last)
-          LAST_NPM_VERSION="$(semver -p $(echo $NPM_VERSIONS_FLATTENED) | tail -n1 || true)"
-          # increment prerelease part of the version
-          PRERELEASE_NPM_VERSION="$(semver --increment prerelease --preid "$SANITIZED_BRANCH" "${LAST_NPM_VERSION}" || true)"
-
-          ### CRATES.IO
-          CRATE_VERSIONS=$(cargo show --json "$PKG_NAME")
-          CRATE_VERSIONS_FILTERED=$(echo $CRATE_VERSIONS | jq -r ".versions[] | .num | select(test(\"$PAT\"))")
-          CRATE_VERSIONS_FLATTENED=$(echo $CRATE_VERSIONS_FILTERED | awk '{print}' ORS=' ')
-          LAST_CRATE_VERSION="$(semver -p $(echo $CRATE_VERSIONS_FLATTENED) | tail -n1 || true)"
-          PRERELEASE_CRATE_VERSION="$(semver --increment prerelease --preid "$SANITIZED_BRANCH" "${LAST_CRATE_VERSION}" || true)"
-
-          ### LOCAL
-          ### (NOTE: the following code assumes that local versions do not contain prerelease suffix; existing suffix will be ignored)
-          # take local Rust version
-          LOCAL_RUST_VERSION="$(toml get "$CARGO_TOML" package.version | tr -d \")"
-          INTERPRETER_RUST_VERSION="$(toml get "$INTERPRETER_CARGO_TOML" package.version | tr -d \")"
-          MAX_RUST_VERSION="$(semver "$LOCAL_RUST_VERSION" "$INTERPRETER_RUST_VERSION" | tail -n1)"
-          LOCAL_RUST_PRERELEASE_VERSION="$(semver --increment prerelease --preid "$SANITIZED_BRANCH" "${MAX_RUST_VERSION}-0")" # added '-0' here to avoid semver erroneously increment patch octet. Any suffix works, '-0' is chosen deliberately.
-
-          ### SAVE FINAL VERSION TO ENV
-          # take the highest version
-          MAX_VERSION="$(semver "$LOCAL_RUST_PRERELEASE_VERSION" "$PRERELEASE_NPM_VERSION" "$PRERELEASE_CRATE_VERSION" | tail -n1)"
-          # save info to env
-          echo "FINAL_VERSION=$MAX_VERSION" | tee -a $GITHUB_ENV
-          echo "PKG_NAME=$PKG_NAME" | tee -a $GITHUB_ENV
-          echo "JS_PKG_NAME=$JS_PKG_NAME" | tee -a $GITHUB_ENV
-
-      - name: Install marine
-        run: cargo install marine
-
-      - name: Set interpreter version to ${{ env.FINAL_VERSION }} before the build
-        run: |
-          (
-            cd air-interpreter
-            toml set Cargo.toml package.version "${{ env.FINAL_VERSION }}" | sponge Cargo.toml
-          )
-          (
-            cd air
-            toml set Cargo.toml package.version "${{ env.FINAL_VERSION }}" | sponge Cargo.toml
-          )
-
-      - name: Build air_interpreter_server.wasm for node
-        run: marine build -p air-interpreter --release --features marine
-        working-directory: air-interpreter
-
-      - name: Copy air_interpreter_server.wasm to interpreter-wasm
-        run: cp target/wasm32-wasi/release/air_interpreter_server.wasm crates/interpreter-wasm/air_interpreter_server.wasm
-
-      - name: Set project name@version to ${{ env.PKG_NAME }}@${{ env.FINAL_VERSION }}
-        run: |
-          toml set Cargo.toml package.version "${{ env.FINAL_VERSION }}" | sponge Cargo.toml
-
-          NAME=$(toml get Cargo.toml package.name | jq -r .)
-          toml set Cargo.toml package.name "${{ env.PKG_NAME }}" | sponge Cargo.toml
-        working-directory: crates/interpreter-wasm
-
-      - name: Login to crates.io
-        run: cargo login ${{ secrets.CRATES_IO_TOKEN }}
-
-      - name: Publish to crates.io
-        run: cargo publish --allow-dirty
-        working-directory: crates/interpreter-wasm
-
-      - name: Install wasm-pack
-        run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
-        continue-on-error: true
-
-      - name: Build npm package
-        working-directory: avm/client
-        run: |
-          ./build_wasm.sh
-          npm i
-          npm run build
-
-      - name: Set version to ${{ env.FINAL_VERSION }}
-        run: yarn version --new-version ${{ env.FINAL_VERSION }} --no-git-tag-version || true
-        working-directory: avm/client
-
-      - name: Setup node
-        uses: actions/setup-node@v1
-        with:
-          node-version: "14"
-          registry-url: "https://registry.npmjs.org"
-
-      - name: Publish to npm registry
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
-        run: npm publish --access public --tag=beta
-        working-directory: avm/client

.github/workflows/snapshot.yml

@@ -0,0 +1,238 @@
+name: Publish snapshots
+
+on:
+  workflow_call:
+    outputs:
+      avm-version:
+        description: "@fluencelabs/avm version"
+        value: ${{ jobs.publish-avm.outputs.version }}
+      air-interpreter-wasm-version:
+        description: "air-interpreter-wasm version"
+        value: ${{ jobs.publish-interpreter.outputs.version }}
+
+env:
+  FORCE_COLOR: true
+
+jobs:
+  wasm:
+    name: "Build air-interpreter-wasm"
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout AquaVM
+        uses: actions/checkout@v3
+
+      - name: Setup Rust toolchain
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+
+      - name: Install marine
+        uses: baptiste0928/cargo-install@v1
+        with:
+          crate: marine
+
+      - name: Build air-interpreter-wasm binary
+        run: marine build --release --features marine
+        working-directory: air-interpreter
+
+      - name: Upload air-interpreter-wasm binary
+        uses: actions/upload-artifact@v3
+        with:
+          name: air-interpreter-wasm
+          path: target/wasm32-wasi/release/air_interpreter_server.wasm
+
+  publish-avm:
+    name: "Publish avm snapshot"
+    runs-on: ubuntu-latest
+
+    needs: wasm
+
+    outputs:
+      version: "${{ steps.build.outputs.version }}"
+
+    permissions:
+      contents: read
+      pull-requests: write
+      id-token: write
+
+    steps:
+      - name: Checkout AquaVM
+        uses: actions/checkout@v3
+
+      - name: Download air-interpreter-wasm binary
+        uses: actions/download-artifact@v3
+        with:
+          name: air-interpreter-wasm
+          path: avm/client/dist/
+
+      - run: mv air_interpreter_server.wasm avm.wasm
+        working-directory: avm/client/dist
+
+      - name: Import secrets
+        uses: hashicorp/vault-action@v2.4.2
+        with:
+          url: https://vault.fluence.dev
+          path: jwt/github
+          role: ci
+          method: jwt
+          jwtGithubAudience: "https://github.com/fluencelabs"
+          jwtTtl: 300
+          exportToken: false
+          secrets: |
+            kv/npm-registry/basicauth/ci token | NODE_AUTH_TOKEN
+
+      - name: Setup node with self-hosted npm registry
+        uses: actions/setup-node@v3
+        with:
+          node-version: "16"
+          registry-url: "https://npm.fluence.dev"
+          cache-dependency-path: avm/client/package-lock.json
+          cache: "npm"
+
+      - run: npm i
+        working-directory: avm/client
+
+      - run: npm run build
+        working-directory: avm/client
+
+      - name: Generate package version
+        id: version
+        run: |
+          SHA=${{ github.event.pull_request.head.sha }}
+          echo "::set-output name=sha::${SHA::7}"
+          echo "::set-output name=branch::${GITHUB_HEAD_REF//[^a-zA-Z0-9-]/-}"
+
+      - name: Set package version
+        id: build
+        env:
+          BRANCH: ${{ steps.version.outputs.branch }}
+          SHA: ${{ steps.version.outputs.sha }}
+          RUN: ${{ github.run_number }}
+          ATTEMPT: ${{ github.run_attempt }}
+        working-directory: avm/client
+        run: |
+          echo "::set-output name=version::$(\
+                npm version prerelease \
+                --no-git-tag-version \
+                --preid ${{ env.BRANCH }}-${{ env.SHA }}-${{ env.RUN }}-${{ env.ATTEMPT }})"
+
+      - name: Publish to self-hosted npm repo
+        working-directory: avm/client
+        run: npm publish --tag snapshot --registry https://npm.fluence.dev
+
+      - name: Find comment in PR
+        uses: peter-evans/find-comment@v1
+        id: comment
+        with:
+          issue-number: "${{ github.event.pull_request.number }}"
+          comment-author: github-actions[bot]
+          body-includes: "## AVM version is"
+
+      - name: Update comment in PR
+        uses: peter-evans/create-or-update-comment@v1
+        env:
+          AVM_VERSION: ${{ steps.build.outputs.version }}
+        with:
+          comment-id: "${{ steps.comment.outputs.comment-id }}"
+          issue-number: "${{ github.event.pull_request.number }}"
+          edit-mode: replace
+          body: |
+            ## AVM version is [${{ env.AVM_VERSION }}](https://npm.fluence.dev/-/web/detail/@fluencelabs/avm/v/${{ env.AVM_VERSION }})
+            To install it run:
+            ```shell
+            npm login --registry https://npm.fluence.dev
+            npm i @fluencelabs/avm@${{ env.AVM_VERSION }} --registry=https://npm.fluence.dev
+            ```
+
+  publish-interpreter:
+    name: "Publish air-interpreter-wasm snapshot"
+    runs-on: ubuntu-latest
+
+    needs: wasm
+
+    outputs:
+      version: "${{ steps.build.outputs.version }}"
+
+    permissions:
+      contents: read
+      pull-requests: write
+      id-token: write
+
+    steps:
+      - name: Checkout AquaVM
+        uses: actions/checkout@v3
+
+      - name: Download air-interpreter-wasm binary
+        uses: actions/download-artifact@v3
+        with:
+          name: air-interpreter-wasm
+
+      - run: mv air_interpreter_server.wasm crates/interpreter-wasm/
+
+      - name: Import secrets
+        uses: hashicorp/vault-action@v2.4.2
+        with:
+          url: https://vault.fluence.dev
+          path: jwt/github
+          role: ci
+          method: jwt
+          jwtGithubAudience: "https://github.com/fluencelabs"
+          jwtTtl: 300
+          exportToken: false
+          secrets: |
+            kv/cargo-registry/users/ci token | CARGO_REGISTRIES_FLUENCE_TOKEN
+
+      - name: Setup Rust toolchain
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+
+      - name: Install cargo-edit
+        uses: baptiste0928/cargo-install@v1
+        with:
+          crate: cargo-edit
+
+      - name: Generate package version
+        id: version
+        run: |
+          SHA=${{ github.event.pull_request.head.sha }}
+          echo "::set-output name=sha::${SHA::7}"
+          echo "::set-output name=branch::${GITHUB_HEAD_REF//[^a-zA-Z0-9-]/-}"
+
+      - name: Set package version
+        id: build
+        env:
+          BRANCH: ${{ steps.version.outputs.branch }}
+          SHA: ${{ steps.version.outputs.sha }}
+          RUN: ${{ github.run_number }}
+          ATTEMPT: ${{ github.run_attempt }}
+        working-directory: crates/interpreter-wasm
+        run: |
+          cargo set-version --bump alpha
+          cargo set-version "$(cargo read-manifest | jq -r .version)-${{ env.BRANCH }}-${{ env.SHA }}-${{ env.RUN }}-${{ env.ATTEMPT }}"
+
+          echo "::set-output name=version::$(cargo read-manifest | jq -r .version)"
+
+      - name: Publish to self-hosted cargo registry
+        working-directory: crates/interpreter-wasm
+        run: cargo publish --registry fluence --allow-dirty
+
+      - name: Find comment in PR
+        uses: peter-evans/find-comment@v1
+        id: comment
+        with:
+          issue-number: "${{ github.event.pull_request.number }}"
+          comment-author: github-actions[bot]
+          body-includes: "## air-interpreter-wasm version is"
+
+      - name: Update comment in PR
+        uses: peter-evans/create-or-update-comment@v1
+        env:
+          VERSION: ${{ steps.build.outputs.version }}
+        with:
+          comment-id: "${{ steps.comment.outputs.comment-id }}"
+          issue-number: "${{ github.event.pull_request.number }}"
+          edit-mode: replace
+          body: |
+            ## air-interpreter-wasm version is [${{ env.VERSION }}](https://crates.fluence.dev/#/crate?name=air-interpreter-wasm&version=${{ env.VERSION }})
+            To install it run:
+            ```shell
+            cargo add air-interpreter-wasm@${{ env.VERSION }} --registry fluence
+            ```

avm/client/build_wasm.sh

@@ -1,9 +1,14 @@
-#!/bin/sh
+#! /usr/bin/env bash
-(
-  cd ../../air-interpreter;
-  marine build  --release --features marine
-)
-
-mkdir -p ./dist/
-cp ../../target/wasm32-wasi/release/air_interpreter_server.wasm ./dist/avm.wasm
 
+# if running in CI avm.wasm has been built already
+if [[ ! -z $GITHUB_SHA ]] && [[ -f dist/avm.wasm ]]; then
+  echo "air-interpreter wasm binary already present"
+  exit 0
+else
+  (
+    cd ../../air-interpreter
+    marine build --release --features marine
+  )
+  mkdir -p dist/
+  cp ../../target/wasm32-wasi/release/air_interpreter_server.wasm dist/avm.wasm
+fi