tests: allow conditionnal execution on pattern

Option can be used to filter out irrelevant tests
usage: ./torture_pki '*ed25519'

Signed-off-by: Aris Adamantiadis <aris@0xbadc0de.be>
Reviewed-by: Andreas Schneider <asn@cryptomilk.org>

tests/client/torture_algorithms.c

@@ -289,7 +289,7 @@ static void torture_algorithms_dh_group1(void **state) {
 }
 int torture_run_tests(void) {
     int rc;
-    const UnitTest tests[] = {
+    UnitTest tests[] = {
         unit_test_setup_teardown(torture_algorithms_aes128_cbc_hmac_sha1, setup, teardown),
         unit_test_setup_teardown(torture_algorithms_aes128_cbc_hmac_sha2_256, setup, teardown),
         unit_test_setup_teardown(torture_algorithms_aes128_cbc_hmac_sha2_512, setup, teardown),
@@ -323,7 +323,7 @@ int torture_run_tests(void) {
     };
 
     ssh_init();
-
+    torture_filter_tests(tests);
     rc = run_tests(tests);
     ssh_finalize();
 

tests/client/torture_auth.c

@@ -421,7 +421,7 @@ static void torture_auth_none_nonblocking(void **state) {
 
 int torture_run_tests(void) {
     int rc;
-    const UnitTest tests[] = {
+    UnitTest tests[] = {
         unit_test_setup_teardown(torture_auth_kbdint, setup, teardown),
         unit_test_setup_teardown(torture_auth_kbdint_nonblocking, setup, teardown),
         unit_test_setup_teardown(torture_auth_password, setup, teardown),
@@ -435,7 +435,7 @@ int torture_run_tests(void) {
     };
 
     ssh_init();
-
+    torture_filter_tests(tests);
     rc = run_tests(tests);
     ssh_finalize();
 

tests/client/torture_connect.c

@@ -139,7 +139,7 @@ static void torture_connect_socket(void **state) {
 
 int torture_run_tests(void) {
     int rc;
-    const UnitTest tests[] = {
+    UnitTest tests[] = {
         unit_test_setup_teardown(torture_connect_nonblocking, setup, teardown),
         unit_test_setup_teardown(torture_connect_double, setup, teardown),
         unit_test_setup_teardown(torture_connect_failure, setup, teardown),
@@ -149,6 +149,7 @@ int torture_run_tests(void) {
 
     ssh_init();
 
+    torture_filter_tests(tests);
     rc = run_tests(tests);
 
     ssh_finalize();

tests/client/torture_forward.c

@@ -81,12 +81,13 @@ static void torture_ssh_forward(void **state)
 int torture_run_tests(void) {
     int rc;
 
-    const UnitTest tests[] = {
+    UnitTest tests[] = {
         unit_test_setup_teardown(torture_ssh_forward, setup, teardown),
     };
 
     ssh_init();
 
+    torture_filter_tests(tests);
     rc = run_tests(tests);
 
     ssh_finalize();

tests/client/torture_knownhosts.c

@@ -282,7 +282,7 @@ static void torture_knownhosts_precheck(void **state) {
 
 int torture_run_tests(void) {
     int rc;
-    const UnitTest tests[] = {
+    UnitTest tests[] = {
         unit_test_setup_teardown(torture_knownhosts_port, setup, teardown),
         unit_test_setup_teardown(torture_knownhosts_fail, setup, teardown),
         unit_test_setup_teardown(torture_knownhosts_other, setup, teardown),
@@ -293,6 +293,7 @@ int torture_run_tests(void) {
 
     ssh_init();
 
+    torture_filter_tests(tests);
     rc = run_tests(tests);
 
     ssh_finalize();

tests/client/torture_proxycommand.c

@@ -42,7 +42,7 @@ static void torture_options_set_proxycommand_notexist(void **state) {
 
 int torture_run_tests(void) {
     int rc;
-    const UnitTest tests[] = {
+    UnitTest tests[] = {
         unit_test_setup_teardown(torture_options_set_proxycommand, setup, teardown),
         unit_test_setup_teardown(torture_options_set_proxycommand_notexist, setup, teardown),
     };
@@ -50,6 +50,7 @@ int torture_run_tests(void) {
 
     ssh_init();
 
+    torture_filter_tests(tests);
     rc = run_tests(tests);
     ssh_finalize();
 

tests/client/torture_request_env.c

@@ -100,12 +100,13 @@ static void torture_request_env(void **state)
 int torture_run_tests(void) {
     int rc;
 
-    const UnitTest tests[] = {
+    UnitTest tests[] = {
         unit_test_setup_teardown(torture_request_env, setup, teardown),
     };
 
     ssh_init();
 
+    torture_filter_tests(tests);
     rc = run_tests(tests);
 
     ssh_finalize();

tests/client/torture_session.c

@@ -99,12 +99,13 @@ static void torture_channel_read_error(void **state) {
 
 int torture_run_tests(void) {
     int rc;
-    const UnitTest tests[] = {
+    UnitTest tests[] = {
         unit_test_setup_teardown(torture_channel_read_error, setup, teardown),
     };
 
     ssh_init();
 
+    torture_filter_tests(tests);
     rc = run_tests(tests);
     ssh_finalize();
 

tests/client/torture_sftp_dir.c

@@ -61,12 +61,13 @@ static void torture_sftp_mkdir(void **state) {
 
 int torture_run_tests(void) {
     int rc;
-    const UnitTest tests[] = {
+    UnitTest tests[] = {
         unit_test_setup_teardown(torture_sftp_mkdir, setup, teardown)
     };
 
     ssh_init();
 
+    torture_filter_tests(tests);
     rc = run_tests(tests);
     ssh_finalize();
 

tests/client/torture_sftp_read.c

@@ -70,12 +70,13 @@ static void torture_sftp_read_blocking(void **state) {
 
 int torture_run_tests(void) {
     int rc;
-    const UnitTest tests[] = {
+    UnitTest tests[] = {
         unit_test_setup_teardown(torture_sftp_read_blocking, setup, teardown)
     };
 
     ssh_init();
 
+    torture_filter_tests(tests);
     rc = run_tests(tests);
     ssh_finalize();
 

tests/client/torture_sftp_static.c

@@ -19,12 +19,13 @@ static void torture_sftp_ext_new(void **state) {
 
 int torture_run_tests(void) {
     int rc;
-    const UnitTest tests[] = {
+    UnitTest tests[] = {
         unit_test(torture_sftp_ext_new),
     };
 
     ssh_init();
 
+    torture_filter_tests(tests);
     rc = run_tests(tests);
     ssh_finalize();
 

tests/cmdline.c

@@ -41,6 +41,7 @@ static error_t parse_opt (int key, char *arg, struct argp_state *state) {
       break;
     case ARGP_KEY_ARG:
       /* End processing here. */
+      arguments->pattern = state->argv[state->next - 1];
       cmdline = &state->argv [state->next - 1];
       state->next = state->argc;
       break;

tests/torture.c

@@ -39,6 +39,8 @@
 #endif
 
 #include "torture.h"
+/* for pattern matching */
+#include "match.c"
 
 #define TORTURE_TESTKEY_PASSWORD "libssh-rocks"
 
@@ -220,6 +222,7 @@ static const char torture_ed25519_testkey_pp[]=
         "-----END OPENSSH PRIVATE KEY-----\n";
 
 static int verbosity = 0;
+static const char *pattern = NULL;
 
 #ifndef _WIN32
 static int _torture_auth_kbdint(ssh_session session,
@@ -650,12 +653,48 @@ int torture_libssh_verbosity(void){
   return verbosity;
 }
 
+void _torture_filter_tests(UnitTest *tests, size_t ntests){
+    size_t i,j;
+    const char *name, *last_name=NULL;
+    if (pattern == NULL){
+        return;
+    }
+    for (i=0; i < ntests; ++i){
+        if(tests[i].function_type == UNIT_TEST_FUNCTION_TYPE_SETUP){
+            /* match on the next test name */
+            name = tests[i+1].name;
+        } else if (tests[i].function_type == UNIT_TEST_FUNCTION_TYPE_TEARDOWN){
+            /* match on the previous test name */
+            name = last_name;
+        } else {
+            name = last_name = tests[i].name;
+        }
+        /*printf("match(%s,%s)\n",name,pattern);*/
+        if (!match_pattern(name, pattern)){
+            for (j = i; j < ntests-1;++j){
+                tests[j]=tests[j+1];
+            }
+            tests[ntests-1].name = NULL;
+            tests[ntests-1].function = NULL;
+            ntests--;
+            --i;
+        }
+    }
+    if (ntests != 0){
+        printf("%d tests left\n",(int)ntests);
+    } else {
+        printf("No matching test left\n");
+    }
+}
+
 int main(int argc, char **argv) {
   struct argument_s arguments;
 
   arguments.verbose=0;
+  arguments.pattern=NULL;
   torture_cmdline_parse(argc, argv, &arguments);
   verbosity=arguments.verbose;
+  pattern=arguments.pattern;
 
   return torture_run_tests();
 }

tests/torture.h

@@ -42,7 +42,7 @@
 
 /* Used by main to communicate with parse_opt. */
 struct argument_s {
-  char *args[2];
+  const char *pattern;
   int verbose;
 };
 
@@ -83,6 +83,9 @@ const char *torture_get_testkey_passphrase(void);
 
 void torture_write_file(const char *filename, const char *data);
 
+#define torture_filter_tests(tests) _torture_filter_tests(tests, sizeof(tests) / sizeof(tests)[0])
+void _torture_filter_tests(UnitTest *tests, size_t ntests);
+
 /*
  * This function must be defined in every unit test file.
  */

tests/unittests/torture_buffer.c

@@ -250,7 +250,7 @@ static void torture_buffer_pack_badformat(void **state){
 
 int torture_run_tests(void) {
     int rc;
-    const UnitTest tests[] = {
+    UnitTest tests[] = {
         unit_test_setup_teardown(torture_growing_buffer, setup, teardown),
         unit_test_setup_teardown(torture_growing_buffer_shifting, setup, teardown),
         unit_test_setup_teardown(torture_buffer_prepend, setup, teardown),
@@ -262,6 +262,7 @@ int torture_run_tests(void) {
     };
 
     ssh_init();
+    torture_filter_tests(tests);
     rc=run_tests(tests);
     ssh_finalize();
     return rc;

tests/unittests/torture_callbacks.c

@@ -98,13 +98,14 @@ static void torture_log_callback(void **state)
 
 int torture_run_tests(void) {
     int rc;
-    const UnitTest tests[] = {
+    UnitTest tests[] = {
         unit_test_setup_teardown(torture_callbacks_size, setup, teardown),
         unit_test_setup_teardown(torture_callbacks_exists, setup, teardown),
         unit_test(torture_log_callback),
     };
 
     ssh_init();
+    torture_filter_tests(tests);
     rc=run_tests(tests);
     ssh_finalize();
     return rc;

tests/unittests/torture_channel.c

@@ -37,11 +37,12 @@ static void torture_channel_select(void **state)
 
 int torture_run_tests(void) {
     int rc;
-    const UnitTest tests[] = {
+    UnitTest tests[] = {
         unit_test(torture_channel_select),
     };
 
     ssh_init();
+    torture_filter_tests(tests);
     rc = run_tests(tests);
     ssh_finalize();
 

tests/unittests/torture_init.c

@@ -15,9 +15,9 @@ static void torture_ssh_init(void **state) {
 }
 
 int torture_run_tests(void) {
-    const UnitTest tests[] = {
+    UnitTest tests[] = {
         unit_test(torture_ssh_init),
     };
-
+    torture_filter_tests(tests);
     return run_tests(tests);
 }

tests/unittests/torture_isipaddr.c

@@ -45,11 +45,12 @@ static void torture_ssh_is_ipaddr(void **state) {
 
 int torture_run_tests(void) {
     int rc;
-    const UnitTest tests[] = {
+    UnitTest tests[] = {
         unit_test(torture_ssh_is_ipaddr)
     };
 
     ssh_init();
+    torture_filter_tests(tests);
     rc=run_tests(tests);
     ssh_finalize();
     return rc;

tests/unittests/torture_keyfiles.c

@@ -240,7 +240,7 @@ static void torture_privatekey_from_file_passphrase(void **state) {
 
 int torture_run_tests(void) {
     int rc;
-    const UnitTest tests[] = {
+    UnitTest tests[] = {
         unit_test_setup_teardown(torture_pubkey_from_file,
                                  setup_rsa_key,
                                  teardown),
@@ -255,6 +255,7 @@ int torture_run_tests(void) {
 
 
     ssh_init();
+    torture_filter_tests(tests);
     rc=run_tests(tests);
     ssh_finalize();
     return rc;

tests/unittests/torture_list.c

@@ -78,13 +78,14 @@ static void torture_ssh_list_prepend(void **state) {
 
 int torture_run_tests(void) {
     int rc;
-    const UnitTest tests[] = {
+    UnitTest tests[] = {
         unit_test(torture_ssh_list_new),
         unit_test(torture_ssh_list_append),
         unit_test(torture_ssh_list_prepend),
     };
 
     ssh_init();
+    torture_filter_tests(tests);
     rc=run_tests(tests);
     ssh_finalize();
     return rc;

tests/unittests/torture_misc.c

@@ -187,7 +187,7 @@ static void torture_timeout_update(void **state){
 
 int torture_run_tests(void) {
     int rc;
-    const UnitTest tests[] = {
+    UnitTest tests[] = {
         unit_test(torture_get_user_home_dir),
         unit_test(torture_basename),
         unit_test(torture_dirname),
@@ -204,6 +204,7 @@ int torture_run_tests(void) {
     };
 
     ssh_init();
+    torture_filter_tests(tests);
     rc=run_tests(tests);
     ssh_finalize();
     return rc;

tests/unittests/torture_options.c

@@ -195,7 +195,7 @@ static void torture_options_proxycommand(void **state) {
 
 int torture_run_tests(void) {
     int rc;
-    const UnitTest tests[] = {
+    UnitTest tests[] = {
         unit_test_setup_teardown(torture_options_set_host, setup, teardown),
         unit_test_setup_teardown(torture_options_get_host, setup, teardown),
         unit_test_setup_teardown(torture_options_set_port, setup, teardown),
@@ -209,6 +209,7 @@ int torture_run_tests(void) {
     };
 
     ssh_init();
+    torture_filter_tests(tests);
     rc=run_tests(tests);
     ssh_finalize();
     return rc;

tests/unittests/torture_pki.c

@@ -1437,7 +1437,7 @@ static void torture_pki_ecdsa_name521(void **state)
 
 int torture_run_tests(void) {
     int rc;
-    const UnitTest tests[] = {
+    UnitTest tests[] = {
         unit_test(torture_pki_keytype),
 
         unit_test(torture_pki_signature),
@@ -1607,6 +1607,7 @@ int torture_run_tests(void) {
     (void)setup_both_keys;
 
     ssh_init();
+    torture_filter_tests(tests);
     rc=run_tests(tests);
     ssh_finalize();
     return rc;

tests/unittests/torture_rand.c

@@ -60,9 +60,10 @@ static void torture_rand_threading(void **state) {
 }
 
 int torture_run_tests(void) {
-    const UnitTest tests[] = {
+    UnitTest tests[] = {
         unit_test_setup_teardown(torture_rand_threading, setup, teardown),
     };
 
+    torture_filter_tests(tests);
     return run_tests(tests);
 }