diff --git a/.github/README.md b/.github/README.md new file mode 100644 index 0000000..6935122 --- /dev/null +++ b/.github/README.md @@ -0,0 +1,39 @@ +# Misc Docs + +## Configuration Files + +### `plugins.json` + +```json +{ + "plugins": { + "": { + "dir": "", + "target": "", + "test": "", + "options": "", + "platforms": [ + { + "key": "" + }, + { + "key": "", + "options": " + }, + { + "key": "", + "options": ", + "plugin_tag": "" + } + } + } + }, + "platforms": { + "": { + "runner": "", + "docker_tag": "", + "asset_tag": "" + } + } +} +``` diff --git a/.github/plugins.json b/.github/plugins.json new file mode 100644 index 0000000..7017c35 --- /dev/null +++ b/.github/plugins.json @@ -0,0 +1,55 @@ +{ + "plugins": { + "wasi_crypto": { + "dir": "wasi_crypto", + "target": "wasmedgePluginWasiCrypto", + "test": "wasiCryptoTests", + "options": "-DWASMEDGE_PLUGIN_WASI_CRYPTO=ON", + "platforms": [ + {"key":"manylinux_2_28_x86_64"}, + {"key":"ubuntu2004_x86_64"} + ] + }, + "wasi_nn-openvino": { + "dir": "wasi_nn", + "target": "wasmedgePluginWasiNN", + "test": "wasiNNTests", + "options": "-DWASMEDGE_PLUGIN_WASI_NN_BACKEND=OpenVINO", + "platforms": [ + {"key":"ubuntu2004_x86_64"} + ] + }, + "wasmedge_stablediffusion": { + "dir": "wasmedge_stablediffusion", + "target": "wasmedgePluginWasmEdgeStableDiffusion", + "test": "wasmedgeStableDiffusionTests", + "options": "-DWASMEDGE_PLUGIN_STABLEDIFFUSION=ON", + "platforms": [ + {"key":"manylinux_2_28_x86_64"}, + {"key":"ubuntu2004_x86_64"}, + { + "key":"ubuntu2004_cuda11", + "options": "-DWASMEDGE_PLUGIN_STABLEDIFFUSION_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES='60;61;70' -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc", + "plugin_tag": "cuda-11" + } + ] + } + }, + "platforms": { + "manylinux_2_28_x86_64": { + "runner": "ubuntu-latest", + "docker_tag": "manylinux_2_28_x86_64-plugins-deps", + "asset_tag": "manylinux_2_28_x86_64" + }, + "ubuntu2004_x86_64": { + "runner": "ubuntu-latest", + "docker_tag": "ubuntu-20.04-build-clang-plugins-deps", + "asset_tag": "ubuntu20.04_x86_64" + }, + "ubuntu2004_cuda11": { + "runner": "ubuntu-latest", + "docker_tag": "ubuntu-20.04-build-gcc-cuda11", + "asset_tag": "ubuntu20.04_x86_64" + } + } +} diff --git a/.github/scripts/parse-plugins.js b/.github/scripts/parse-plugins.js new file mode 100644 index 0000000..3459f19 --- /dev/null +++ b/.github/scripts/parse-plugins.js @@ -0,0 +1,27 @@ +module.exports.parse = (config) => { + let map = new Map(); + for (const platform_key of Object.keys(config.platforms)) { + map.set(platform_key, []); + } + for (const [plugin_key, plugin] of Object.entries(config.plugins)) { + for (const platform of plugin.platforms) { + if (!(platform.key in config.platforms)) + continue; + let copy = { ...plugin, ...config.platforms[platform.key] }; + delete copy.platforms; + copy.plugin = plugin_key; + copy.plugin_tag = platform.plugin_tag; + copy.options = [plugin.options, platform.options].join(" "); + map.get(platform.key).push(copy); + } + } + return Object.fromEntries(map); +}; + +if (require.main === module) { + const { parse } = module.exports; + const fs = require("fs"); + const s = fs.readFileSync("plugins.json"); + const o = JSON.parse(s); + console.log(JSON.stringify(parse(o))); +} diff --git a/.github/workflows/build-on-linux.yml b/.github/workflows/build-on-linux.yml new file mode 100644 index 0000000..2a1b946 --- /dev/null +++ b/.github/workflows/build-on-linux.yml @@ -0,0 +1,72 @@ +--- +name: Build on Linux + +on: + workflow_call: + inputs: + plugins: + type: string + required: true + version: + type: string + required: true + +permissions: {} + +jobs: + build: + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(inputs.plugins) }} + name: ${{ matrix.plugin }} + runs-on: ${{ matrix.runner }} + container: + image: wasmedge/wasmedge:${{ matrix.docker_tag }} + env: + bin: lib${{ matrix.target }}.so + target: ${{ matrix.test }} # TODO: use matrix.target on release + test_dir: build/test/plugins/${{ matrix.dir }} + output_dir: build/plugins/${{ matrix.dir }} + defaults: + run: + shell: bash + steps: + - id: var + run: | + prefix="WasmEdge-plugin-${{ matrix.plugin }}" + if [[ -n "${{ matrix.plugin_tag }}" ]]; then + prefix="${prefix}-${{ matrix.plugin_tag }}" + fi + echo "artifact=${prefix}-${{ inputs.version }}-${{ matrix.asset_tag }}.tar.gz" >> "${GITHUB_OUTPUT}" + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + repository: 'WasmEdge/WasmEdge' # TODO: checkout plugins from this repository + - name: Ensure git safe directory + run: | + git config --global --add safe.directory "$(pwd)" + - name: Build ${{ matrix.plugin }} + run: | + # TODO: Disable BUILD_TESTS on release + # TODO: Enable CXX11_ABI if not manylinux_2_28 + cmake -Bbuild -GNinja \ + -DCMAKE_BUILD_TYPE=Release \ + -DWASMEDGE_BUILD_TESTS=ON \ + -DWASMEDGE_BUILD_TOOLS=OFF \ + -DWASMEDGE_USE_LLVM=OFF \ + -DWASMEDGE_USE_CXX11_ABI=OFF \ + "-DOPENSSL_ROOT_DIR=${OpenSSL_DIR}" \ + ${{ matrix.options }} + cmake --build build --target "${target}" + + cp -f "${output_dir}/${bin}" "${bin}" + tar -zcvf "${{ steps.var.outputs.artifact }}" "${bin}" + - name: Test ${{ matrix.plugin }} + run: | + cd "${test_dir}" && "./${target}" + - name: Upload ${{ steps.var.outputs.artifact }} + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.var.outputs.artifact }} + path: ${{ steps.var.outputs.artifact }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..9c59b47 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,83 @@ +--- +name: Build + +on: + pull_request: + branches: [main] + +permissions: {} + +jobs: + get-versions: + uses: ./.github/workflows/get-versions.yml + + parse-plugins: + uses: ./.github/workflows/parse-plugins.yml + + linux: + needs: + - get-versions + - parse-plugins + strategy: + fail-fast: false + matrix: + include: + - name: manylinux_2_28 (x86_64) + plugins: ${{ fromJSON(needs.parse-plugins.outputs.plugins).manylinux_2_28_x86_64 }} + - name: ubuntu20.04 (x86_64) + plugins: ${{ fromJSON(needs.parse-plugins.outputs.plugins).ubuntu2004_x86_64 }} + - name: ubuntu20.04 (x86_64) with CUDA 11.3 + plugins: ${{ fromJSON(needs.parse-plugins.outputs.plugins).ubuntu2004_cuda11 }} + name: ${{ matrix.name }} + uses: ./.github/workflows/build-on-linux.yml + with: + plugins: ${{ toJSON(matrix.plugins) }} + version: ${{ needs.get-versions.outputs.version }} + secrets: inherit + + check-artifacts: + if: ${{ !cancelled() }} # TODO: check output from `get-versions` and `parse-plugins` + needs: + - get-versions + - parse-plugins + - linux + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/github-script@v7 + with: + script: | + const fs = require("fs"); + const raw = JSON.parse(fs.readFileSync(".github/plugins.json")); + + core.summary.addHeading('Build Summary'); + + let table = [ + [{data: 'plugin/platform', header: true}], + ]; + let platforms = []; + Object.entries(raw.platforms).forEach(([key, platform]) => { + table[0].push({data: key, header: true}); + platforms.push([platform.plugin_tag, platform.asset_tag].filter((tag) => tag !== undefined)); + }); + + let artifacts = await github + .paginate(github.rest.actions.listWorkflowRunArtifacts, { + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId, + }); + + Object.keys(raw.plugins).forEach((plugin) => { + let row = [{data: plugin, header: true}]; + let filtered = artifacts.filter((artifact) => artifact.name.includes(plugin)); + platforms.forEach((tags) => { + row.push({ + data: filtered.some((artifact) => tags.every((tag) => artifact.name.includes(tag))) ? ':ok:' : ':x:' + }); + }); + table.push(row); + }); + + core.summary.addTable(table); + core.summary.write() diff --git a/.github/workflows/get-versions.yml b/.github/workflows/get-versions.yml new file mode 100644 index 0000000..4dacdf7 --- /dev/null +++ b/.github/workflows/get-versions.yml @@ -0,0 +1,25 @@ +--- +name: Get versions + +on: + workflow_call: + outputs: + version: + value: ${{ jobs.get-versions.outputs.version }} + +jobs: + get-versions: + name: Retrieve version information + runs-on: ubuntu-latest + outputs: + version: ${{ steps.parse.outputs.version }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + repository: 'WasmEdge/WasmEdge' # TODO: checkout plugins from this repository + - id: parse + run: | + git fetch --tags --force + VERSION=$(git describe --match '[0-9].[0-9]*' --tag) + echo "version=${VERSION}" >> "${GITHUB_OUTPUT}" diff --git a/.github/workflows/parse-plugins.yml b/.github/workflows/parse-plugins.yml new file mode 100644 index 0000000..2984f84 --- /dev/null +++ b/.github/workflows/parse-plugins.yml @@ -0,0 +1,52 @@ +--- +name: Parse + +on: + pull_request: + branches: [main] + paths: + - '.github/plugins.json' + - '.github/scripts/parse-plugins.js' + - '.github/workflows/parse-plugins.yml' + + workflow_call: + inputs: + workflow_call: # workaround to distinguish ${{ github.event }} + type: boolean + default: true + outputs: + plugins: + value: ${{ jobs.parse.outputs.plugins }} + +jobs: + parse: + name: Parse configuration file + runs-on: ubuntu-latest + outputs: + plugins: ${{ steps.readfile.outputs.plugins }} + steps: + - uses: actions/checkout@v4 + - id: readfile + uses: actions/github-script@v7 + with: + result-encoding: string + script: | + const { parse } = require(".github/scripts/parse-plugins.js"); + const fs = require("fs"); + const s = fs.readFileSync(".github/plugins.json"); + const config = parse(JSON.parse(s)); + core.setOutput("plugins", config); + + test: + if: ${{ !inputs.workflow_call }} + needs: parse + name: Self-testing + runs-on: ubuntu-latest + steps: + - name: Get the first target from ubuntu2004_cuda11 + run: | + echo "target=${{ fromJSON(needs.parse.outputs.plugins).ubuntu2004_cuda11[0].target }}" >> "${GITHUB_ENV}" + - name: Check target + run: | + echo "${target}" + test "${target}" = 'wasmedgePluginWasmEdgeStableDiffusion'