MDEV-37193 Clean up spider tests

Clean up spider tests by removing excessive $VARIABLE abstractions.
Indentations are preserved to minimise diff. After this change only
one disabled test remains with the following regexp:

$ (cd ../src && git grep -l "eval .*\$\(MASTER\|CHILD\|SLAVE\)[A-Z0-9_]*_\(COMMENT\|TABLES\)") | grep spider.*\.test$
storage/spider/mysql-test/spider/bugfix/t/wait_timeout.test

The patch is created with the following steps:

Step 1. hack mysqltest.cc to log statements

modified   client/mysqltest.cc
@@ -1745,17 +1745,23 @@ void verbose_msg(const char *fmt, ...)
   DBUG_ENTER("verbose_msg");
   DBUG_PRINT("enter", ("format: %s", fmt));

-  if (!verbose)
-    DBUG_VOID_RETURN;
+  /*
+    if (!verbose)
+      DBUG_VOID_RETURN;
+   */

   fflush(stdout);
   va_start(args, fmt);
-  fprintf(stderr, "mysqltest: ");
+  /*
+    fprintf(stderr, "mysqltest: ");
+   */
   if (cur_file && cur_file != file_stack)
     fprintf(stderr, "In included file \"%s\": ",
             cur_file->file_name);
-  if (start_lineno != 0)
-    fprintf(stderr, "At line %u: ", start_lineno);
+  /*
+    if (start_lineno != 0)
+      fprintf(stderr, "At line %u: ", start_lineno);
+   */
   vfprintf(stderr, fmt, args);
   fprintf(stderr, "\n");
   va_end(args);
@@ -10412,6 +10418,22 @@ int main(int argc, char **argv)

     if (ok_to_do)
     {
+      /* only log current file / do not log included files */
+      if (!cur_file || cur_file == file_stack)
+      {
+        if (command->type == Q_EMPTY_LINE)
+          verbose_msg("%s", "");
+        /* ignore if; eval will be handled after var subst */
+        else if (command->type != Q_IF && command->type != Q_END_BLOCK &&
+                 command->type != Q_EVAL)
+          /*
+            read_command_buf instead of command->query because only
+            the former retains leading "--"
+          */
+          verbose_msg("%s%s", read_command_buf,
+                      strncmp("--", read_command_buf, 2) &&
+                      strncmp("#", read_command_buf, 1) ? ";" : "");
+      }
       command->last_argument= command->first_argument;
       processed = 1;
       /* Need to remember this for handle_error() */
@@ -10595,6 +10617,14 @@ int main(int argc, char **argv)
         /* Restore settings */
 	display_result_vertically= old_display_result_vertically;

+        /* only log current file / do not log included files */
+        if (!cur_file || cur_file == file_stack)
+        {
+          /* print evaled query */
+          if (command->type == Q_EVAL)
+            verbose_msg("%s;", command->eval_query.str);
+        }
+
 	break;
       }
       case Q_SEND:

Statements are printed in mysql-test/var/log/mysqltest.log between two
lines:

"Start processing test commands from '$test_file_name' ..."

and

"... Done processing test commands."

Step 2. Overwrite the test files with logged statements and run the
tests again to check they pass (not all pass, see below)

(cd ../src && git grep -l "eval \$CHILD") | grep \.test$ | sed "s/^.*spider\//spider\//g" | sed "s/\/t\//./g" | sed "s/\.test$//g" | xargs -i sh -c './mysql-test/mtr {} && emacs --batch --load /home/ycp/source/mariadb-tools/mdev-37193.el --eval "(my-mdev-37193)" && ./mysql-test/mtr {}'

where my-mdev-37193 is as follows:

(defun my-mdev-37193 ()
  (let ((log-file
         (file-name-concat default-directory
                           "mysql-test/var/log/mysqltest.log"))
        test-file-name beg end)
    (if (file-exists-p log-file)
        (with-temp-buffer
          (insert-file-contents log-file)
          (goto-char (point-min))
          (re-search-forward
           "^Start processing test commands from '\\(.*\\)' ...$")
          (setq test-file-name (match-string 1))
          (beginning-of-line 2)
          (setq beg (point))
          (re-search-forward "^... Done processing test commands.$")
          (beginning-of-line 1)
          (setq end (point))
          (write-region beg end test-file-name)
          (message "Wrote %s." test-file-name))
      (message "Log file does not exist (test disabled?): %s" log-file))))

In my run, out of the 79 cleaned tests, about 8 failed:

  21606:spider/bg.spider_fixes                   [ fail ]
  22463:spider/bugfix.mdev_20100                 [ fail ]
  22734:spider/bugfix.mdev_21884                 [ fail ]
  23876:spider/regression/e1121.direct_join_by_pkey_key [ fail ]
  24077:spider/regression/e1121.direct_join_by_pkey_pkey [ fail ]
  25653:spider.partition_mrr                     [ fail ]
  26388:spider.spider_fixes                      [ fail ]
  26970:spider.udf_pushdown                      [ fail ]

Step 3. Remove all blank line removals in the diff

The diff could contain removal of blank lines which could affect
readability of tests arranged in logical blocks. this can be fixed
with the following:

git diff > backup.diff
git diff --ignore-blank-lines -U1 > changes.diff
git reset --hard
git apply -- changes.diff

Step 4. Manually Fix up the 8 failing tests above, which was relatively
straightforward.

Step 5. Fix sockets. Sockets need to be changed back to vars because
their paths depends on those of the vardir.

Do this

(cd ../src && git grep -l "mysql-test/var/tmp/.*\\.sock") | grep \.test$ | xargs  -i sh -c 'emacs --batch --load /home/ycp/source/mariadb-tools/mdev-37193.el --eval "(my-mdev-37193-fix-sockets \"../src/{}\")"'

where my-mdev-37193-fix-sockets is defined as follows:

(defun my-mdev-37193--translate-sock (m n)
  (pcase m
    ("1" "$MASTER_1_MYSOCK")
    ("2" (format "$CHILD2_%s_MYSOCK" n))
    ("3" (format "$CHILD3_%s_MYSOCK" n))
    ("4" (format "$SLAVE1_%s_MYSOCK" n))))

(defun my-mdev-37193--fix-sockets ()
  "Fix sockets in the current buffer"
  (goto-char (point-min))
  (while (re-search-forward
          "socket \".*mysqld\\.\\(.\\)\\.\\(.\\)\\.sock\"" nil t)
    (replace-match (format "socket \"%s\""
                           (my-mdev-37193--translate-sock
                            (match-string 1)
                            (match-string 2))))
    (re-search-backward ";$")
    (beginning-of-line 2)
    (insert "eval\n")))

(defun my-mdev-37193-fix-sockets (file)
  "Fix sockets in FILE"
  (with-temp-buffer
    (insert-file-contents file)
    (my-mdev-37193--fix-sockets)
    (write-file file)))

Subsequent small fixup of spider.ha and spider/bugfix.ha are needed
because they involve strings of two sockets instead of one.

Step 6. Clean up the "main" inc files by removing the obsolete vars.
This can be done manually with the help of grep as there are not many
such files. Note that there are still about 100 remaining .inc files
matching relevant regexp (using the command below), but they are all
"specialised", i.e. $VARs used in foo.test are declared in foo.inc,
instead of some generic files several --source's away. So the
readability is not as bad, and they are minor nuisances that can be
easily fixed on demand and on-the-fly when needed.

$ (cd ../src && git grep -l "let \$\(MASTER\|CHILD\|SLAVE\)[A-Z0-9_]*_\(COMMENT\|TABLES\)") | grep spider.*\.inc | sed "s/^/..\/src\//" | wc -l
99

Step 7. Further clean up white spaces to reduce diff and ease merging:

git checkout -b tmp
git diff -U0 -w --no-color | git apply --cached --ignore-whitespace --unidiff-zero -
git checkout .
git commit

Author: mariadb-YuchenPei