[Backport 8.7] Integration test cleanup (#1841 (#1844)Co-authored-by: Josh Mock <[email protected]>

* Fix bad sysctl command in README

* Add --suite and --test flags to integration tests

So we can run a single suite or a single test without having to edit
any code.

* Drop several skipped integration tests

Many of these skips are no longer necessary. Didn't do an exhaustive
check of all skipped tests, so this is just a start.

* Simplify cleanup make target

Co-authored-by: Josh Mock <[email protected]>

Author: github-actions[bot]

Makefile

@@ -4,8 +4,7 @@ integration-setup: integration-cleanup
 
 .PHONY: integration-cleanup
 integration-cleanup:
-	docker stop instance || true
-	docker volume rm instance-rest-test-data || true
+	docker container rm --force --volumes instance || true
 
 .PHONY: integration
 integration: integration-setup

test/integration/README.md

@@ -37,7 +37,7 @@ make integration
 
 If Elasticsearch doesn't come up, run `make integration-cleanup` and then `DETACH=false .ci/run-elasticsearch.sh` manually to read the startup logs.
 
-If you get an error about `vm.max_map_count` being too low, run `sudo sysctl -w vm.max_map_count=262144` to update the setting until the next reboot, or `sudo sysctl -w vm.max_map_count=262144 | sudo tee -a /etc/sysctl.conf` to update the setting permanently.
+If you get an error about `vm.max_map_count` being too low, run `sudo sysctl -w vm.max_map_count=262144` to update the setting until the next reboot, or `sudo sysctl -w vm.max_map_count=262144; echo 'vm.max_map_count=262144' | sudo tee -a /etc/sysctl.conf` to update the setting permanently.
 
 ### Exit on the first failure
 

test/integration/index.js

@@ -30,7 +30,6 @@ const yaml = require('js-yaml')
 const minimist = require('minimist')
 const ms = require('ms')
 const { Client } = require('../../index')
-const { kProductCheck } = require('@elastic/transport/lib/symbols')
 const build = require('./test-runner')
 const { sleep } = require('./helper')
 const createJunitReporter = require('./reporter')
@@ -44,7 +43,8 @@ const MAX_FILE_TIME = 1000 * 30
 const MAX_TEST_TIME = 1000 * 3
 
 const options = minimist(process.argv.slice(2), {
-  boolean: ['bail']
+  boolean: ['bail'],
+  string: ['suite', 'test'],
 })
 
 const freeSkips = {
@@ -56,10 +56,6 @@ const freeSkips = {
   '/free/cluster.desired_nodes/20_dry_run.yml': ['*'],
   '/free/cluster.prevalidate_node_removal/10_basic.yml': ['*'],
 
-  '/free/health/30_feature.yml': ['*'],
-  '/free/health/40_useractions.yml': ['*'],
-  '/free/health/40_diagnosis.yml': ['Diagnosis'],
-
   // the v8 client never sends the scroll_id in querystring,
   // the way the test is structured causes a security exception
   'free/scroll/10_basic.yml': ['Body params override query string'],
@@ -70,9 +66,6 @@ const freeSkips = {
   'free/cat.allocation/10_basic.yml': ['*'],
   'free/cat.snapshots/10_basic.yml': ['Test cat snapshots output'],
 
-  // TODO: remove this once 'arbitrary_key' is implemented
-  // https://github.com/elastic/elasticsearch/pull/41492
-  'indices.split/30_copy_settings.yml': ['*'],
   'indices.stats/50_disk_usage.yml': ['Disk usage stats'],
   'indices.stats/60_field_usage.yml': ['Field usage stats'],
 
@@ -99,35 +92,13 @@ const platinumDenyList = {
   'api_key/11_invalidation.yml': ['Test invalidate api key by realm name'],
   'analytics/histogram.yml': ['Histogram requires values in increasing order'],
 
-  // this two test cases are broken, we should
-  // return on those in the future.
-  'analytics/top_metrics.yml': [
-    'sort by keyword field fails',
-    'sort by string script fails'
-  ],
-
-  'cat.aliases/10_basic.yml': ['Empty cluster'],
-  'index/10_with_id.yml': ['Index with ID'],
-  'indices.get_alias/10_basic.yml': ['Get alias against closed indices'],
-  'indices.get_alias/20_empty.yml': ['Check empty aliases when getting all aliases via /_alias'],
-  'text_structure/find_structure.yml': ['*'],
-
-  // https://github.com/elastic/elasticsearch/pull/39400
-  'ml/jobs_crud.yml': ['Test put job with id that is already taken'],
-
   // object keys must me strings, and `0.0.toString()` is `0`
   'ml/evaluate_data_frame.yml': [
     'Test binary_soft_classifition precision',
     'Test binary_soft_classifition recall',
     'Test binary_soft_classifition confusion_matrix'
   ],
 
-  // it gets random failures on CI, must investigate
-  'ml/set_upgrade_mode.yml': [
-    'Attempt to open job when upgrade_mode is enabled',
-    'Setting upgrade mode to disabled from enabled'
-  ],
-
   // The cleanup fails with a index not found when retrieving the jobs
   'ml/get_datafeed_stats.yml': ['Test get datafeed stats when total_search_time_ms mapping is missing'],
   'ml/bucket_correlation_agg.yml': ['Test correlation bucket agg simple'],
@@ -209,13 +180,6 @@ const platinumDenyList = {
 
   // start should be a string in the yaml test
   'platinum/ml/start_stop_datafeed.yml': ['*'],
-
-  // health API not yet supported
-  '/platinum/health/10_usage.yml': ['*'],
-
-  // ML update_trained_model_deployment not supported yet
-  '/platinum/ml/3rd_party_deployment.yml': ['Test update deployment'],
-  '/platinum/ml/update_trained_model_deployment.yml': ['Test with unknown model id']
 }
 
 function runner (opts = {}) {
@@ -227,8 +191,6 @@ function runner (opts = {}) {
     }
   }
   const client = new Client(options)
-  // TODO: remove the following line once https://github.com/elastic/elasticsearch/issues/82358 is fixed
-  client.transport[kProductCheck] = null
   log('Loading yaml suite')
   start({ client, isXPack: opts.isXPack })
     .catch(err => {
@@ -333,13 +295,21 @@ async function start ({ client, isXPack }) {
       }
 
       const cleanPath = file.slice(file.lastIndexOf(apiName))
+
+      // skip if --suite CLI arg doesn't match
+      if (options.suite && !cleanPath.endsWith(options.suite)) continue
+
       log('    ' + cleanPath)
       const junitTestSuite = junitTestSuites.testsuite(apiName.slice(1) + ' - ' + cleanPath)
 
       for (const test of tests) {
         const testTime = now()
         const name = Object.keys(test)[0]
+
+        // skip setups, teardowns and anything that doesn't match --test flag when present
         if (name === 'setup' || name === 'teardown') continue
+        if (options.test && !name.endsWith(options.test)) continue
+
         const junitTestCase = junitTestSuite.testcase(name)
 
         stats.total += 1
@@ -439,6 +409,8 @@ if (require.main === module) {
 }
 
 const shouldSkip = (isXPack, file, name) => {
+  if (options.suite || options.test) return false
+
   let list = Object.keys(freeSkips)
   for (let i = 0; i < list.length; i++) {
     const freeTest = freeSkips[list[i]]