[llvm][cas] Implement a CAS stress tester

This is a basic tester for CAS ObjectStore that will insert random data
into CAS and validate with several configurations randomly generated.
It will check:

* multi-threaded insertion
* multi-process insertion
* try randomly kill the subprocesses that are inserting data

And make sure it doesn't leave CAS in an invalid state. Suggested usage:
```
LLVM_CAS_LOG=2 llvm-cas-test --cas-path=... --print-config
```

Author: cachemeifyoucan

llvm/tools/llvm-cas-test/CMakeLists.txt

@@ -0,0 +1,7 @@
+set(LLVM_LINK_COMPONENTS
+  CAS
+  Support
+  )
+add_llvm_tool(llvm-cas-test
+  llvm-cas-test.cpp
+  )

llvm/tools/llvm-cas-test/llvm-cas-test.cpp

@@ -0,0 +1,287 @@
+#include "llvm/ADT/BitmaskEnum.h"
+#include "llvm/CAS/ActionCache.h"
+#include "llvm/CAS/BuiltinUnifiedCASDatabases.h"
+#include "llvm/CAS/ObjectStore.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Program.h"
+#include "llvm/Support/RandomNumberGenerator.h"
+#include "llvm/Support/ThreadPool.h"
+
+using namespace llvm;
+using namespace llvm::cas;
+
+// Options from commandline.
+static std::string CASPath;
+static bool GenData = false;
+static bool PrintConfig = false;
+static bool ForceKill = false;
+static unsigned OptNumShards = 0;
+static unsigned OptTreeDepth = 0;
+static unsigned OptNumChildren = 0;
+static unsigned OptDataLength = 0;
+static unsigned OptPrecentFile = 0;
+static const char *Argv0 = nullptr;
+
+// Default size to be 100MB.
+static uint64_t SizeLimit = 100;
+// Default timeout 180s.
+static uint64_t Timeout = 180;
+
+enum CASFuzzingSettings : uint8_t {
+  DEFAULT = 0,
+  FORK = 1,                   // CAS Data filling happens in subprocesses.
+  CHECK_TERMINATION = 1 << 1, // Try kill the subprocess when it fills the data.
+  PRUNE_CAS = 1 << 2,         // Prune the CAS after the test.
+
+  LAST = UINT8_MAX, // Enum is randomly generated, use MAX to cover all inputs.
+  LLVM_MARK_AS_BITMASK_ENUM(LAST)
+};
+
+struct Config {
+  CASFuzzingSettings Settings = DEFAULT;
+  uint8_t NumShards;
+  uint8_t NumChildren;
+  uint8_t TreeDepth;
+  uint16_t DataLength;
+  uint16_t PrecentFile;
+
+  static constexpr unsigned MaxShards = 20;
+  static constexpr unsigned MaxChildren = 32;
+  static constexpr unsigned MaxDepth = 8;
+  static constexpr unsigned MaxDataLength = 1024 * 4;
+
+  void constraint() {
+    // reduce the size of parameter if they are too big.
+    NumShards = NumShards % MaxShards;
+    NumChildren = NumChildren % MaxChildren;
+    TreeDepth = TreeDepth % MaxDepth;
+    DataLength = DataLength % MaxDataLength;
+    PrecentFile = PrecentFile % 100;
+
+    if (ForceKill) {
+      Settings |= FORK;
+      Settings |= CHECK_TERMINATION;
+    }
+  }
+
+  bool extendToFile(uint8_t Seed) const {
+    return ((float)Seed / (float)UINT8_MAX) > ((float)PrecentFile / 100.0f);
+  }
+
+  void init() {
+    NumShards = OptNumShards ? OptNumShards : MaxShards;
+    NumChildren = OptNumChildren ? OptNumChildren : MaxChildren;
+    TreeDepth = OptTreeDepth ? OptTreeDepth : MaxDepth;
+    DataLength = OptDataLength ? OptDataLength : MaxDataLength;
+    PrecentFile = OptPrecentFile;
+  }
+
+  void appendCommandLineOpts(std::vector<std::string> &Cmd) {
+    Cmd.push_back("--num-shards=" + utostr(NumShards));
+    Cmd.push_back("--num-children=" + utostr(NumChildren));
+    Cmd.push_back("--tree-depth=" + utostr(TreeDepth));
+    Cmd.push_back("--data-length=" + utostr(DataLength));
+    Cmd.push_back("--precent-file=" + utostr(PrecentFile));
+  }
+
+  void dump() {
+    llvm::errs() << "## Configuration:"
+                 << " Fork: " << (bool)(Settings & FORK)
+                 << " Kill: " << (bool)(Settings & CHECK_TERMINATION)
+                 << " Prune: " << (bool)(Settings & PRUNE_CAS)
+                 << " NumShards: " << (unsigned)NumShards
+                 << " TreeDepth: " << (unsigned)TreeDepth
+                 << " NumChildren: " << (unsigned)NumChildren
+                 << " DataLength: " << (unsigned)DataLength
+                 << " PrecentFile: " << (unsigned)PrecentFile << "\n";
+  }
+};
+
+static void parseOptions(int Argc, char **Argv) {
+  Argv0 = Argv[0];
+
+  for (int I = 0; I < Argc; ++I) {
+    StringRef Arg = Argv[I];
+    // option must start with `--`.
+    if (!Arg.consume_front("--"))
+      continue;
+
+    // flags.
+    if (Arg == "gen-data") {
+      GenData = true;
+      continue;
+    }
+    if (Arg == "print-config") {
+      PrintConfig = true;
+      continue;
+    }
+    if (Arg == "force-kill") {
+      ForceKill = true;
+      continue;
+    }
+
+    // options that take an argument.
+    auto Opt = Arg.split('=');
+    if (Opt.first == "cas-path")
+      CASPath = Opt.second.str();
+    else if (Opt.first == "num-shards")
+      to_integer(Opt.second, OptNumShards);
+    else if (Opt.first == "num-children")
+      to_integer(Opt.second, OptNumChildren);
+    else if (Opt.first == "tree-depth")
+      to_integer(Opt.second, OptTreeDepth);
+    else if (Opt.first == "data-length")
+      to_integer(Opt.second, OptDataLength);
+    else if (Opt.first == "precent-file")
+      to_integer(Opt.second, OptPrecentFile);
+    else if (Opt.first == "cas-size-limit")
+      to_integer(Opt.second, SizeLimit);
+    else if (Opt.first == "timeout")
+      to_integer(Opt.second, Timeout);
+  }
+}
+
+// fill the CAS with random data of specified tree depth and children numbers.
+static void fillData(ObjectStore &CAS, const Config &Conf) {
+  ExitOnError ExitOnErr("llvm-cas-fuzzer fill data: ");
+  DefaultThreadPool ThreadPool(hardware_concurrency());
+  std::atomic<uint64_t> NumCreated = 0;
+  for (size_t I = 0; I != Conf.NumShards; ++I) {
+    ThreadPool.async([&] {
+      std::vector<ObjectRef> Refs;
+      for (unsigned Depth = 0; Depth < Conf.TreeDepth; ++Depth) {
+        unsigned NumNodes = (Conf.TreeDepth - Depth + 1) * Conf.NumChildren + 1;
+        std::vector<ObjectRef> Created;
+        Created.reserve(NumNodes);
+        ArrayRef<ObjectRef> PreviouslyCreated(Refs);
+        for (unsigned I = 0; I < NumNodes; ++I) {
+          std::vector<char> Data(Conf.DataLength);
+          getRandomBytes(Data.data(), Data.size());
+          // Use the first byte that generated to decide if we should make it
+          // 64KB bigger and force that into a file based storage.
+          if (Conf.extendToFile(Data[0]))
+            Data.resize(64LL * 1024LL + Conf.DataLength);
+
+          if (Depth == 0) {
+            auto Ref = ExitOnErr(CAS.store({}, Data));
+            Created.push_back(Ref);
+          } else {
+            auto Parent = PreviouslyCreated.slice(I, Conf.NumChildren);
+            auto Ref = ExitOnErr(CAS.store(Parent, Data));
+            Created.push_back(Ref);
+          }
+          ++NumCreated;
+        }
+        Refs.swap(Created);
+      }
+    });
+  }
+  ThreadPool.wait();
+}
+
+static int genData() {
+  ExitOnError ExitOnErr("llvm-cas-test --gen-data: ");
+
+  Config Conf;
+  Conf.init();
+
+  auto DB = ExitOnErr(cas::createOnDiskUnifiedCASDatabases(CASPath));
+  fillData(*DB.first, Conf);
+
+  return 0;
+}
+
+static int runOneTest() {
+  ExitOnError ExitOnErr("llvm-cas-test: ");
+
+  Config Conf;
+  getRandomBytes(&Conf, sizeof(Conf));
+  Conf.constraint();
+
+  if (PrintConfig)
+    Conf.dump();
+
+  // Start with fresh log.
+  static constexpr StringLiteral LogFile = "v1.log";
+  SmallString<256> LogPath(CASPath);
+  llvm::sys::path::append(LogPath, LogFile);
+  llvm::sys::fs::remove(LogPath);
+
+  auto DB = ExitOnErr(cas::createOnDiskUnifiedCASDatabases(CASPath));
+  auto &CAS = DB.first;
+
+  // Size limit in MB.
+  ExitOnErr(CAS->setSizeLimit(SizeLimit * 1024 * 1024));
+  if (Conf.Settings & FORK) {
+    // fill data using sub processes.
+    std::string MainExe = sys::fs::getMainExecutable(Argv0, &Argv0);
+    std::vector<std::string> Args = {MainExe, "--gen-data",
+                                     "--cas-path=" + CASPath};
+    Conf.appendCommandLineOpts(Args);
+
+    std::vector<StringRef> Cmd;
+    for_each(Args, [&Cmd](const std::string &Arg) { Cmd.push_back(Arg); });
+
+    std::vector<sys::ProcessInfo> Subprocesses;
+    for (int I = 0; I < Conf.NumShards; ++I) {
+      auto SP = sys::ExecuteNoWait(MainExe, Cmd, std::nullopt);
+      if (SP.Pid != 0)
+        Subprocesses.push_back(SP);
+    }
+
+    if (Conf.Settings & CHECK_TERMINATION) {
+      for_each(Subprocesses, [](auto &P) {
+        // Wait 1 second and killed the process.
+        auto WP = sys::Wait(P, 1);
+        if (WP.ReturnCode)
+          llvm::errs() << "subprocess killed successfully\n";
+      });
+    } else {
+      for_each(Subprocesses, [](auto &P) { sys::Wait(P, std::nullopt); });
+    }
+
+  } else {
+    // in-process fill data.
+    fillData(*CAS, Conf);
+  }
+
+  // validate and prune in the end.
+  ExitOnErr(CAS->validate(true));
+
+  if (Conf.Settings & PRUNE_CAS)
+    ExitOnErr(CAS->pruneStorageData());
+
+  return 0;
+}
+
+static int runTest() {
+  auto Start = std::chrono::steady_clock::now();
+  std::chrono::seconds Duration(Timeout);
+
+  while (std::chrono::steady_clock::now() - Start < Duration) {
+    if (int Res = runOneTest())
+      return Res;
+  }
+
+  std::chrono::duration_cast<std::chrono::seconds>(
+      std::chrono::steady_clock::now() - Start)
+      .count();
+
+  return 0;
+}
+
+int main(int argc, char **argv) {
+  parseOptions(argc, argv);
+
+  if (CASPath.empty()) {
+    llvm::errs() << "llvm-cas-test requires --cas-path= option\n";
+    return 1;
+  }
+
+  if (GenData) {
+    genData();
+    return 0;
+  }
+
+  return runTest();
+}