wip schema query batching

activerecord/CHANGELOG.md

@@ -1,3 +1,26 @@
+*   Return `[]`/`nil` instead of raising when querying primary keys for a
+    missing PostgreSQL table.
+
+    Aligns PostgreSQL with the MySQL adapter's existing behavior.
+
+    *Hartley McGuire*
+
+*   Batch schema cache dump queries for MySQL and PostgreSQL.
+
+    `SchemaCache#add_all` now loads primary keys, columns, and indexes for
+    every cached table in three batched queries instead of one query per table
+    per type, speeding up the schema cache dump by roughly 25% on databases
+    with many tables.
+
+    *Hartley McGuire*
+
+*   Use `information_schema` to query MySQL indexes.
+
+    Results will now be returned in alphabetical order by index name, which
+    may result in a one-time reordering of indexes in `schema.rb`.
+
+    *Hartley McGuire*
+
 *   Deprecate the `insert_returning` option in PostgreSQL database
     configurations, and the `PostgreSQLAdapter#use_insert_returning?` method.
 

activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb

@@ -81,6 +81,12 @@ def indexes(table_name)
         raise NotImplementedError, "#indexes is not implemented"
       end
 
+      def indexes_for_tables(table_names) # :nodoc:
+        table_names.each_with_object({}) do |table_name, hash|
+          hash[table_name.to_s] = indexes(table_name)
+        end
+      end
+
       # Checks to see if an index exists on a table for a given index definition.
       #
       #   # Check an index exists
@@ -111,6 +117,12 @@ def columns(table_name)
         end
       end
 
+      def columns_for_tables(table_names) # :nodoc:
+        table_names.each_with_object({}) do |table_name, hash|
+          hash[table_name.to_s] = columns(table_name)
+        end
+      end
+
       # Checks to see if a column exists in a given table.
       #
       #   # Check a column exists
@@ -147,6 +159,12 @@ def primary_key(table_name)
         pk
       end
 
+      def primary_keys_for_tables(table_names) # :nodoc:
+        table_names.each_with_object({}) do |table_name, hash|
+          hash[table_name.to_s] = primary_keys(table_name)
+        end
+      end
+
       # Creates a new table with the name +table_name+. +table_name+ may either
       # be a String or a Symbol.
       #

activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb

@@ -690,17 +690,23 @@ def show_variable(name)
 
       def primary_keys(table_name) # :nodoc:
         raise ArgumentError unless table_name.present?
+        primary_keys_for_tables([table_name]).each_value.first || []
+      end
 
-        scope = quoted_scope(table_name)
+      def primary_keys_for_tables(table_names) # :nodoc:
+        return {} if table_names.empty?
 
-        query_values(<<~SQL)
-          SELECT column_name
+        result = query_all(<<~SQL)
+          SELECT table_name AS 'table_name', column_name AS 'column_name'
           FROM information_schema.statistics
           WHERE index_name = 'PRIMARY'
-            AND table_schema = #{scope[:schema]}
-            AND table_name = #{scope[:name]}
-          ORDER BY seq_in_index
+            AND (#{information_schema_scope_sql(table_names)})
+          ORDER BY table_name, seq_in_index
         SQL
+
+        result.each_with_object(Hash.new { |h, k| h[k] = [] }) do |row, primary_keys_by_table|
+          primary_keys_by_table[row["table_name"]] << row["column_name"]
+        end
       end
 
       def case_sensitive_comparison(attribute, value) # :nodoc:
@@ -1116,32 +1122,62 @@ def configure_connection
         end
 
         def column_definitions(table_name) # :nodoc:
-          fields = query_all("SHOW FULL FIELDS FROM #{quote_table_name(table_name)}")
+          fields = column_definitions_for_tables([table_name]).each_value.first
+          raise ActiveRecord::StatementInvalid.new("Could not find table '#{table_name}'", connection_pool: @pool) unless fields
+          fields
+        end
 
-          update_fields_for_mariadb(table_name, fields) if mariadb?
+        def column_definitions_for_tables(table_names) # :nodoc:
+          return {} if table_names.empty?
+
+          result = query_all(<<~SQL)
+            SELECT COLUMN_NAME AS 'Field',
+                   COLUMN_TYPE AS 'Type',
+                   COLLATION_NAME AS 'Collation',
+                   IS_NULLABLE AS 'Null',
+                   COLUMN_KEY AS 'Key',
+                   COLUMN_DEFAULT AS 'Default',
+                   EXTRA AS 'Extra',
+                   PRIVILEGES AS 'Privileges',
+                   COLUMN_COMMENT AS 'Comment',
+                   TABLE_NAME AS 'table_name'
+            FROM information_schema.columns
+            WHERE #{information_schema_scope_sql(table_names)}
+            ORDER BY table_name, ORDINAL_POSITION
+          SQL
 
-          fields
+          fields_by_table = result.each_with_object(Hash.new { |h, k| h[k] = [] }) do |row, hash|
+            hash[row.delete("table_name")] << row
+          end
+
+          if mariadb?
+            fields_by_table.each_value do |fields|
+              update_fields_for_mariadb(fields)
+            end
+          end
+
+          fields_by_table
         end
 
-        def update_fields_for_mariadb(table_name, fields)
+        def update_fields_for_mariadb(fields)
           has_function_default_candidate = fields.any? do |field|
             default = field["Default"]
             default&.match?(/[a-zA-Z_]\w*\(/) && !/\ACURRENT_TIMESTAMP/i.match?(default)
           end
+          return unless has_function_default_candidate
+
+          # MariaDB quotes string literal defaults in information_schema.COLUMNS.COLUMN_DEFAULT
+          # (e.g. `'uuid()'`) but leaves function/expression defaults unquoted (e.g. `uuid()`),
+          # unlike SHOW FULL FIELDS which reports both as `uuid()`. Use that to distinguish
+          # function defaults from literal strings that merely look like function calls, without
+          # parsing SHOW CREATE TABLE output.
+          fields.each do |field|
+            default = field["Default"]
+            next unless default&.match?(/[a-zA-Z_]\w*\(/)
+            next if /\ACURRENT_TIMESTAMP/i.match?(default)
+            next if default.start_with?("'")
 
-          if has_function_default_candidate
-            table_info = create_table_info(table_name)
-            fields.each do |field|
-              default = field["Default"]
-              next unless default&.match?(/[a-zA-Z_]\w*\(/)
-              next if /\ACURRENT_TIMESTAMP/i.match?(default)
-
-              field_name = field["Field"]
-              match = table_info&.match(/`#{field_name}` .+ DEFAULT ('|\d+|[A-z]+)/)
-              if match && match[1].match?(/\A[A-z]/)
-                field["Extra"] = "DEFAULT_GENERATED"
-              end
-            end
+            field["Extra"] = "DEFAULT_GENERATED"
           end
         end
 

activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb

@@ -6,14 +6,42 @@ module MySQL
       module SchemaStatements # :nodoc:
         # Returns an array of indexes for the given table.
         def indexes(table_name)
-          indexes = []
+          indexes_for_tables([table_name]).each_value.first || []
+        end
+
+        def indexes_for_tables(table_names) # :nodoc:
+          return {} if table_names.empty?
+
+          optional_columns = +""
+          optional_columns << ", EXPRESSION AS 'Expression'" if supports_expression_index?
+          if supports_disabling_indexes?
+            optional_columns << (mariadb? ? ", IF(ignored = 'NO', 'YES', 'NO') AS 'enabled'" : ", is_visible AS 'enabled'")
+          end
+
+          result = query_all(<<~SQL)
+            SELECT TABLE_NAME AS 'Table', INDEX_NAME AS 'Key_name',
+                   NON_UNIQUE AS 'Non_unique', SEQ_IN_INDEX AS 'Seq_in_index',
+                   COLUMN_NAME AS 'Column_name', COLLATION AS 'Collation',
+                   SUB_PART AS 'Sub_part', LOWER(INDEX_TYPE) AS 'Index_type',
+                   INDEX_COMMENT AS 'Index_comment'#{optional_columns}
+            FROM information_schema.statistics
+            WHERE (#{information_schema_scope_sql(table_names)})
+              AND index_name != 'PRIMARY'
+            ORDER BY table_name, index_name, seq_in_index
+          SQL
+
+          indexes_by_table = Hash.new { |h, k| h[k] = [] }
+
           current_index = nil
-          query_all("SHOW KEYS FROM #{quote_table_name(table_name)}").each do |row|
-            if current_index != row["Key_name"]
-              next if row["Key_name"] == "PRIMARY" # skip the primary key
-              current_index = row["Key_name"]
 
-              mysql_index_type = row["Index_type"].downcase.to_sym
+          result.each do |row|
+            table = row["Table"]
+            index_key = [table, row["Key_name"]]
+
+            if current_index != index_key
+              current_index = index_key
+
+              mysql_index_type = row["Index_type"].to_sym
               case mysql_index_type
               when :fulltext, :spatial
                 index_type = mysql_index_type
@@ -22,7 +50,7 @@ def indexes(table_name)
               end
 
               index = [
-                row["Table"],
+                table,
                 row["Key_name"],
                 row["Non_unique"].to_i == 0,
                 [],
@@ -34,48 +62,57 @@ def indexes(table_name)
               ]
 
               if supports_disabling_indexes?
-                index[-1][:enabled] = mariadb? ? row["Ignored"] == "NO" : row["Visible"] == "YES"
+                index[-1][:enabled] = row["enabled"] == "YES"
               end
 
-              indexes << index
+              indexes_by_table[table] << index
             end
 
+            index = indexes_by_table[table].last
+
             if expression = row["Expression"]
               expression = expression.gsub("\\'", "'")
               expression = +"(#{expression})" unless expression.start_with?("(")
-              indexes.last[-2] << expression
-              indexes.last[-1][:expressions] ||= {}
-              indexes.last[-1][:expressions][expression] = expression
-              indexes.last[-1][:orders][expression] = :desc if row["Collation"] == "D"
+              index[-2] << expression
+              index[-1][:expressions] ||= {}
+              index[-1][:expressions][expression] = expression
+              index[-1][:orders][expression] = :desc if row["Collation"] == "D"
             else
-              indexes.last[-2] << row["Column_name"]
-              indexes.last[-1][:lengths][row["Column_name"]] = row["Sub_part"].to_i if row["Sub_part"]
-              indexes.last[-1][:orders][row["Column_name"]] = :desc if row["Collation"] == "D"
+              index[-2] << row["Column_name"]
+              index[-1][:lengths][row["Column_name"]] = row["Sub_part"].to_i if row["Sub_part"]
+              index[-1][:orders][row["Column_name"]] = :desc if row["Collation"] == "D"
             end
           end
 
-          indexes.map do |index|
-            options = index.pop
+          indexes_by_table.transform_values! do |table_indexes|
+            table_indexes.map! do |index|
+              options = index.pop
 
-            if expressions = options.delete(:expressions)
-              orders = options.delete(:orders)
-              lengths = options.delete(:lengths)
+              if expressions = options.delete(:expressions)
+                orders = options.delete(:orders)
+                lengths = options.delete(:lengths)
 
-              columns = index[-1].to_h { |name|
-                [ name.to_sym, expressions[name] || +quote_column_name(name) ]
-              }
+                columns = index[-1].to_h { |name|
+                  [ name.to_sym, expressions[name] || +quote_column_name(name) ]
+                }
 
-              index[-1] = add_options_for_index_columns(
-                columns, order: orders, length: lengths
-              ).values.join(", ")
+                index[-1] = add_options_for_index_columns(
+                  columns, order: orders, length: lengths
+                ).values.join(", ")
+              end
+              MySQL::IndexDefinition.new(*index, **options)
             end
-            MySQL::IndexDefinition.new(*index, **options)
           end
-        rescue StatementInvalid => e
-          if e.message.match?(/Table '.+' doesn't exist/)
-            []
-          else
-            raise
+
+          indexes_by_table
+        end
+
+        def columns_for_tables(table_names) # :nodoc:
+          definitions_by_table = column_definitions_for_tables(table_names)
+          definitions_by_table.each_with_object({}) do |(table, definitions), hash|
+            hash[table] = definitions.map do |field|
+              new_column_from_field(table, field, definitions)
+            end
           end
         end
 
@@ -197,7 +234,15 @@ def create_table_definition(name, **options)
           def new_column_from_field(table_name, field, _definitions)
             type_metadata = fetch_type_metadata(field["Type"], field["Extra"])
             default, default_function = field["Default"], nil
-
+            # MariaDB reports the COLUMN_DEFAULT of a nullable column with no
+            # default as the string "NULL" rather than SQL NULL; normalize it so
+            # the column is treated as having no default. MySQL already returns
+            # SQL NULL for a no-default column, and leaves a literal "NULL"
+            # string default unquoted, so this must be MariaDB-only — otherwise
+            # MySQL's literal "NULL" defaults would be wiped to nil. A literal
+            # "NULL" string default on MariaDB is quoted ('NULL') and is handled
+            # by the strip branch below.
+            default = nil if mariadb? && default == "NULL"
             if type_metadata.type == :datetime && /\ACURRENT_TIMESTAMP(?:\([0-6]?\))?\z/i.match?(default)
               default = "#{default} ON UPDATE #{default}" if /on update CURRENT_TIMESTAMP/i.match?(field["Extra"])
               default, default_function = nil, default
@@ -209,9 +254,15 @@ def new_column_from_field(table_name, field, _definitions)
                 default = default.gsub("\\'", "'")
                 default, default_function = nil, default
               end
-            elsif type_metadata.type == :text && default&.start_with?("'")
-              # strip and unescape quotes
-              default = default[1...-1].gsub("\\'", "'")
+            elsif mariadb? && default&.start_with?("'")
+              # MariaDB quotes string literal defaults in information_schema
+              # COLUMN_DEFAULT (e.g. `'abc'`, `'O''Connor'`) and escapes
+              # embedded single quotes — by doubling them in varchar defaults
+              # (`''`) but with a backslash in text defaults (`\'`). MySQL leaves
+              # string defaults unquoted, so this only runs on MariaDB. Strip the
+              # wrapping quotes and undo both escape forms so the default matches
+              # what SHOW FULL FIELDS reported.
+              default = default[1...-1].gsub("''", "'").gsub("\\'", "'")
             end
 
             MySQL::Column.new(
@@ -283,6 +334,19 @@ def extract_schema_qualified_name(string)
             [schema, name]
           end
 
+          def information_schema_scope_sql(table_names)
+            by_schema = table_names.each_with_object(Hash.new { |h, k| h[k] = [] }) do |name, h|
+              schema, tbl = extract_schema_qualified_name(name)
+              h[schema] << tbl
+            end
+
+            by_schema.map do |schema, names|
+              schema_sql = schema ? quote(schema) : "database()"
+              names_sql = names.map { |n| quote(n) }.join(", ")
+              "(table_schema = #{schema_sql} AND table_name IN (#{names_sql}))"
+            end.join(" OR ")
+          end
+
           def type_with_size_to_sql(type, size)
             case size&.to_s
             when nil, "tiny", "medium", "long"