scripts/test: Add option to run all test cases in a single OSv instance

Previously, scripts/test.py had no option to do that. It launched an OSv
instance for each test case.
Terribly slow PCs like mine took a bunch of time to run all test cases
through 'make check'.

Then let's take advantage of testrunner.so which will use a single OSv instance
to run all test cases, consequently boosting the speed considerably.
Let's also change testunner.so to conform our needs, e.g. blacklist.

To run this fast check, do: scripts/test.py --single;

Results show that this option is about 2.5x faster than the current one.

By now, let's not use this approach as the default version given that its
output has to be better formatted.

Signed-off-by: Raphael S. Carvalho <raphaelsc@cloudius-systems.com>
Signed-off-by: Pekka Enberg <penberg@cloudius-systems.com>

scripts/test.py

@@ -78,15 +78,24 @@ def run_test(name):
         sys.stdout.write(" OK  (%.3f s)\n" % duration)
         sys.stdout.flush()
 
+def run_tests_in_single_instance():
+    blacklist_tests = ' '.join(blacklist)
+
+    args = ["-s", "-e", "/testrunner.so -b %s" % (blacklist_tests)]
+    subprocess.call(["./scripts/run.py"] + args)
+
 def run_tests():
     start = time.time()
 
-    for test in tests:
-        if not test in blacklist:
-            run_test(test)
-        else:
-            sys.stdout.write("  TEST %-25s SKIPPED\n" % test)
-            sys.stdout.flush()
+    if cmdargs.single:
+        run_tests_in_single_instance()
+    else:
+        for test in tests:
+            if not test in blacklist:
+                run_test(test)
+            else:
+                sys.stdout.write("  TEST %-25s SKIPPED\n" % test)
+                sys.stdout.flush()
 
     end = time.time()
 
@@ -103,5 +112,6 @@ if (__name__ == "__main__"):
     parser = argparse.ArgumentParser(prog='test')
     parser.add_argument("-v", "--verbose", action="store_true", help="verbose test output")
     parser.add_argument("-r", "--repeat", action="store_true", help="repeat until test fails")
+    parser.add_argument("-s", "--single", action="store_true", help="run all tests in a single OSv instance")
     cmdargs = parser.parse_args()
     main()

tests/testrunner.cc

@@ -53,11 +53,28 @@ int check_path(char *path)
 	return 1;
 }
 
+bool is_test_in_blacklist(const char *name, int argc, char **argv)
+{
+	/* Start from the index 2 as 1 would be -b */
+	for (int i = 2; i < argc; i++) {
+		if (!strcmp(argv[i], name)) {
+			return true;
+		}
+	}
+
+	return false;
+}
+
 int main(int argc, char **argv)
 {
 	char path[PATH_MAX];
+	bool blacklist = false;
 
-	if (argc == 1) {
+	if (argc > 1 && !strcmp(argv[1], "-b")) {
+		blacklist = true;
+	}
+
+	if (argc == 1 || blacklist) {
 		DIR *dir = opendir(TESTDIR);
 		struct dirent *d;
 
@@ -72,7 +89,10 @@ int main(int argc, char **argv)
 				continue;
 
 			if (strncmp(d->d_name, "tst-", 4) != 0)
-			    continue;
+				continue;
+
+			if (blacklist && is_test_in_blacklist(d->d_name, argc, argv))
+				continue;
 
 			snprintf(path, PATH_MAX, "%s/%s", TESTDIR, d->d_name);
 			if (!check_path(path))