[llvm][cas] Implement a CAS stress tester

This is a basic tester for CAS ObjectStore that will insert random data
into CAS and validate with several configurations randomly generated.
It will check:

* multi-threaded insertion
* multi-process insertion
* try randomly kill the subprocesses that are inserting data

And make sure it doesn't leave CAS in an invalid state. Suggested usage:
```
LLVM_CAS_LOG=2 llvm-cas-test --cas-path=... --print-config
```

Author: cachemeifyoucan

llvm/test/CMakeLists.txt

@@ -79,8 +79,9 @@ set(LLVM_TEST_DEPENDS
           llvm-bitcode-strip
           llvm-c-test
           llvm-cas
-          llvm-cas-object-format
           llvm-cas-dump
+          llvm-cas-object-format
+          llvm-cas-test
           llvm-cat
           llvm-cfi-verify
           llvm-cgdata

llvm/test/tools/llvm-cas/stress.test

@@ -0,0 +1,8 @@
+REQUIRES: expensive_checks
+
+RUN: rm -rf %t.cas
+RUN: llvm-cas-test --cas-path %t.cas --timeout 60
+
+// Clear the CAS if passed.
+RUN: rm -rf %t.cas
+

llvm/tools/llvm-cas-test/CMakeLists.txt

@@ -0,0 +1,7 @@
+set(LLVM_LINK_COMPONENTS
+  CAS
+  Support
+  )
+add_llvm_tool(llvm-cas-test
+  llvm-cas-test.cpp
+  )

llvm/tools/llvm-cas-test/llvm-cas-test.cpp

@@ -0,0 +1,252 @@
+//===- llvm-cas-test.cpp - CAS stress tester ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "llvm/ADT/BitmaskEnum.h"
+#include "llvm/CAS/ActionCache.h"
+#include "llvm/CAS/BuiltinUnifiedCASDatabases.h"
+#include "llvm/CAS/ObjectStore.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Program.h"
+#include "llvm/Support/RandomNumberGenerator.h"
+#include "llvm/Support/ThreadPool.h"
+
+using namespace llvm;
+using namespace llvm::cas;
+
+static cl::opt<std::string>
+    CASPath("cas-path", cl::desc("CAS path on disk for testing"), cl::Required);
+static cl::opt<bool> GenData("gen-data",
+                             cl::desc("generate data into CAS only"));
+static cl::opt<bool>
+    PrintConfig("print-config",
+                cl::desc("print randomly generated configuration"));
+static cl::opt<bool>
+    ForceKill("force-kill",
+              cl::desc("force kill subprocess to test termination"));
+static cl::opt<unsigned>
+    OptNumShards("num-shards", cl::desc("number of shards"), cl::init(0));
+static cl::opt<unsigned> OptTreeDepth("tree-depth", cl::desc("tree depth"),
+                                      cl::init(0));
+static cl::opt<unsigned> OptNumChildren("num-children",
+                                        cl::desc("number of child nodes"),
+                                        cl::init(0));
+static cl::opt<unsigned> OptDataLength("data-length", cl::desc("data length"),
+                                       cl::init(0));
+static cl::opt<unsigned> OptPrecentFile(
+    "precent-file",
+    cl::desc("percentage of nodes that is long enough to be file based"),
+    cl::init(0));
+
+// Default size to be 100MB.
+static cl::opt<uint64_t>
+    SizeLimit("size-limit", cl::desc("CAS size limit (in MB)"), cl::init(100));
+// Default timeout 180s.
+static cl::opt<uint64_t>
+    Timeout("timeout", cl::desc("test timeout (in seconds)"), cl::init(180));
+
+enum CASFuzzingSettings : uint8_t {
+  DEFAULT = 0,
+  FORK = 1,                   // CAS Data filling happens in subprocesses.
+  CHECK_TERMINATION = 1 << 1, // Try kill the subprocess when it fills the data.
+
+  LAST = UINT8_MAX, // Enum is randomly generated, use MAX to cover all inputs.
+  LLVM_MARK_AS_BITMASK_ENUM(LAST)
+};
+
+struct Config {
+  CASFuzzingSettings Settings = DEFAULT;
+  uint8_t NumShards;
+  uint8_t NumChildren;
+  uint8_t TreeDepth;
+  uint16_t DataLength;
+  uint16_t PrecentFile;
+
+  static constexpr unsigned MaxShards = 20;
+  static constexpr unsigned MaxChildren = 32;
+  static constexpr unsigned MaxDepth = 8;
+  static constexpr unsigned MaxDataLength = 1024 * 4;
+
+  void constraint() {
+    // reduce the size of parameter if they are too big.
+    NumShards = NumShards % MaxShards;
+    NumChildren = NumChildren % MaxChildren;
+    TreeDepth = TreeDepth % MaxDepth;
+    DataLength = DataLength % MaxDataLength;
+    PrecentFile = PrecentFile % 100;
+
+    if (ForceKill) {
+      Settings |= FORK;
+      Settings |= CHECK_TERMINATION;
+    }
+  }
+
+  bool extendToFile(uint8_t Seed) const {
+    return ((float)Seed / (float)UINT8_MAX) > ((float)PrecentFile / 100.0f);
+  }
+
+  void init() {
+    NumShards = OptNumShards ? OptNumShards : MaxShards;
+    NumChildren = OptNumChildren ? OptNumChildren : MaxChildren;
+    TreeDepth = OptTreeDepth ? OptTreeDepth : MaxDepth;
+    DataLength = OptDataLength ? OptDataLength : MaxDataLength;
+    PrecentFile = OptPrecentFile;
+  }
+
+  void appendCommandLineOpts(std::vector<std::string> &Cmd) {
+    Cmd.push_back("--num-shards=" + utostr(NumShards));
+    Cmd.push_back("--num-children=" + utostr(NumChildren));
+    Cmd.push_back("--tree-depth=" + utostr(TreeDepth));
+    Cmd.push_back("--data-length=" + utostr(DataLength));
+    Cmd.push_back("--precent-file=" + utostr(PrecentFile));
+  }
+
+  void dump() {
+    llvm::errs() << "## Configuration:"
+                 << " Fork: " << (bool)(Settings & FORK)
+                 << " Kill: " << (bool)(Settings & CHECK_TERMINATION)
+                 << " NumShards: " << (unsigned)NumShards
+                 << " TreeDepth: " << (unsigned)TreeDepth
+                 << " NumChildren: " << (unsigned)NumChildren
+                 << " DataLength: " << (unsigned)DataLength
+                 << " PrecentFile: " << (unsigned)PrecentFile << "\n";
+  }
+};
+
+// fill the CAS with random data of specified tree depth and children numbers.
+static void fillData(ObjectStore &CAS, const Config &Conf) {
+  ExitOnError ExitOnErr("llvm-cas-fuzzer fill data: ");
+  DefaultThreadPool ThreadPool(hardware_concurrency());
+  std::atomic<uint64_t> NumCreated = 0;
+  for (size_t I = 0; I != Conf.NumShards; ++I) {
+    ThreadPool.async([&] {
+      std::vector<ObjectRef> Refs;
+      for (unsigned Depth = 0; Depth < Conf.TreeDepth; ++Depth) {
+        unsigned NumNodes = (Conf.TreeDepth - Depth + 1) * Conf.NumChildren + 1;
+        std::vector<ObjectRef> Created;
+        Created.reserve(NumNodes);
+        ArrayRef<ObjectRef> PreviouslyCreated(Refs);
+        for (unsigned I = 0; I < NumNodes; ++I) {
+          std::vector<char> Data(Conf.DataLength);
+          getRandomBytes(Data.data(), Data.size());
+          // Use the first byte that generated to decide if we should make it
+          // 64KB bigger and force that into a file based storage.
+          if (Conf.extendToFile(Data[0]))
+            Data.resize(64LL * 1024LL + Conf.DataLength);
+
+          if (Depth == 0) {
+            auto Ref = ExitOnErr(CAS.store({}, Data));
+            Created.push_back(Ref);
+          } else {
+            auto Parent = PreviouslyCreated.slice(I, Conf.NumChildren);
+            auto Ref = ExitOnErr(CAS.store(Parent, Data));
+            Created.push_back(Ref);
+          }
+          ++NumCreated;
+        }
+        Refs.swap(Created);
+      }
+    });
+  }
+  ThreadPool.wait();
+}
+
+static int genData() {
+  ExitOnError ExitOnErr("llvm-cas-test --gen-data: ");
+
+  Config Conf;
+  Conf.init();
+
+  auto DB = ExitOnErr(cas::createOnDiskUnifiedCASDatabases(CASPath));
+  fillData(*DB.first, Conf);
+
+  return 0;
+}
+
+static int runOneTest(const char *argv0) {
+  ExitOnError ExitOnErr("llvm-cas-test: ");
+
+  Config Conf;
+  getRandomBytes(&Conf, sizeof(Conf));
+  Conf.constraint();
+
+  if (PrintConfig)
+    Conf.dump();
+
+  // Start with fresh log.
+  static constexpr StringLiteral LogFile = "v1.log";
+  SmallString<256> LogPath(CASPath);
+  llvm::sys::path::append(LogPath, LogFile);
+  llvm::sys::fs::remove(LogPath);
+
+  auto DB = ExitOnErr(cas::createOnDiskUnifiedCASDatabases(CASPath));
+  auto &CAS = DB.first;
+
+  // Size limit in MB.
+  ExitOnErr(CAS->setSizeLimit(SizeLimit * 1024 * 1024));
+  if (Conf.Settings & FORK) {
+    // fill data using sub processes.
+    std::string MainExe = sys::fs::getMainExecutable(argv0, &CASPath);
+    std::vector<std::string> Args = {MainExe, "--gen-data",
+                                     "--cas-path=" + CASPath};
+    Conf.appendCommandLineOpts(Args);
+
+    std::vector<StringRef> Cmd;
+    for_each(Args, [&Cmd](const std::string &Arg) { Cmd.push_back(Arg); });
+
+    std::vector<sys::ProcessInfo> Subprocesses;
+    for (int I = 0; I < Conf.NumShards; ++I) {
+      auto SP = sys::ExecuteNoWait(MainExe, Cmd, std::nullopt);
+      if (SP.Pid != 0)
+        Subprocesses.push_back(SP);
+    }
+
+    if (Conf.Settings & CHECK_TERMINATION) {
+      for_each(Subprocesses, [](auto &P) {
+        // Wait 1 second and killed the process.
+        auto WP = sys::Wait(P, 1);
+        if (WP.ReturnCode)
+          llvm::errs() << "subprocess killed successfully\n";
+      });
+    } else {
+      for_each(Subprocesses, [](auto &P) { sys::Wait(P, std::nullopt); });
+    }
+
+  } else {
+    // in-process fill data.
+    fillData(*CAS, Conf);
+  }
+
+  // validate and prune in the end.
+  ExitOnErr(CAS->validate(true));
+  ExitOnErr(CAS->pruneStorageData());
+
+  return 0;
+}
+
+int main(int argc, char **argv) {
+  cl::ParseCommandLineOptions(argc, argv, "llvm-cas CAS tool\n");
+
+  if (GenData) {
+    genData();
+    return 0;
+  }
+
+  auto Start = std::chrono::steady_clock::now();
+  std::chrono::seconds Duration(Timeout);
+
+  while (std::chrono::steady_clock::now() - Start < Duration) {
+    if (int Res = runOneTest(argv[0]))
+      return Res;
+  }
+
+  std::chrono::duration_cast<std::chrono::seconds>(
+      std::chrono::steady_clock::now() - Start)
+      .count();
+
+  return 0;
+}