[Web] Set TOKENIZERS_PARALLELISM to false for HFTokenizer (#42)

This PR sets TOKENIZERS_PARALLELISM to false when compiling for web, which huggingface/tokenizers reads in runtime.

Author: CharlieFRuan

src/huggingface_tokenizer.cc

@@ -15,7 +15,11 @@ namespace tokenizers {
  */
 class HFTokenizer : public Tokenizer {
  public:
-  explicit HFTokenizer(TokenizerHandle handle) : handle_(handle) {}
+  explicit HFTokenizer(TokenizerHandle handle) : handle_(handle) {
+    #ifdef COMPILE_WASM_RUNTIME
+    setenv("TOKENIZERS_PARALLELISM", "false", true);
+    #endif
+  }
 
   HFTokenizer(const HFTokenizer&) = delete;
   HFTokenizer(HFTokenizer&& other) { std::swap(other.handle_, handle_); }

web/build.sh

@@ -5,7 +5,7 @@ rustup target add wasm32-unknown-emscripten
 
 mkdir -p build
 cd build
-emcmake cmake ../.. -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="-O3"
+emcmake cmake ../.. -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="-O3 -DCOMPILE_WASM_RUNTIME"
 emmake make tokenizers_cpp tokenizers_c sentencepiece-static -j8
 cd ..
 

web/tests/src/index.ts

@@ -48,8 +48,21 @@ async function testLlamaTokenizer() {
   }
 }
 
+// Without COMPILE_WASM_RUNTIME, this triggers parallel processing, leading to error
+async function testBertTokenizer() {
+  console.log("Bert Tokenizer");
+  const modelBuffer = await (await
+    fetch("https://huggingface.co/Snowflake/snowflake-arctic-embed-l/raw/main/tokenizer.json")
+  ).arrayBuffer();
+  const tok = await Tokenizer.fromJSON(modelBuffer);
+  const text = "What is the capital of Canada?";
+  const ids = tok.encode(text);
+  console.log(ids);
+}
+
 async function main() {
   await testJSONTokenizer()
+  await testBertTokenizer();
   await testLlamaTokenizer()
 }