diff --git a/activerecord/CHANGELOG.md b/activerecord/CHANGELOG.md index 47482d874e368..a2a9c88521d37 100644 --- a/activerecord/CHANGELOG.md +++ b/activerecord/CHANGELOG.md @@ -1,3 +1,26 @@ +* Return `[]`/`nil` instead of raising when querying primary keys for a + missing PostgreSQL table. + + Aligns PostgreSQL with the MySQL adapter's existing behavior. + + *Hartley McGuire* + +* Batch schema cache dump queries for MySQL and PostgreSQL. + + `SchemaCache#add_all` now loads primary keys, columns, and indexes for + every cached table in three batched queries instead of one query per table + per type, speeding up the schema cache dump by roughly 25% on databases + with many tables. + + *Hartley McGuire* + +* Use `information_schema` to query MySQL indexes. + + Results will now be returned in alphabetical order by index name, which + may result in a one-time reordering of indexes in `schema.rb`. + + *Hartley McGuire* + * Deprecate the `insert_returning` option in PostgreSQL database configurations, and the `PostgreSQLAdapter#use_insert_returning?` method. diff --git a/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb b/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb index e65989ecbff4d..e78b091630b42 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb @@ -81,6 +81,12 @@ def indexes(table_name) raise NotImplementedError, "#indexes is not implemented" end + def indexes_for_tables(table_names) # :nodoc: + table_names.each_with_object({}) do |table_name, hash| + hash[table_name.to_s] = indexes(table_name) + end + end + # Checks to see if an index exists on a table for a given index definition. # # # Check an index exists @@ -111,6 +117,12 @@ def columns(table_name) end end + def columns_for_tables(table_names) # :nodoc: + table_names.each_with_object({}) do |table_name, hash| + hash[table_name.to_s] = columns(table_name) + end + end + # Checks to see if a column exists in a given table. # # # Check a column exists @@ -147,6 +159,12 @@ def primary_key(table_name) pk end + def primary_keys_for_tables(table_names) # :nodoc: + table_names.each_with_object({}) do |table_name, hash| + hash[table_name.to_s] = primary_keys(table_name) + end + end + # Creates a new table with the name +table_name+. +table_name+ may either # be a String or a Symbol. # diff --git a/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb b/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb index d1b24ba69a6b4..990e23806b3fb 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb @@ -690,17 +690,23 @@ def show_variable(name) def primary_keys(table_name) # :nodoc: raise ArgumentError unless table_name.present? + primary_keys_for_tables([table_name]).each_value.first || [] + end - scope = quoted_scope(table_name) + def primary_keys_for_tables(table_names) # :nodoc: + return {} if table_names.empty? - query_values(<<~SQL) - SELECT column_name + result = query_all(<<~SQL) + SELECT table_name AS 'table_name', column_name AS 'column_name' FROM information_schema.statistics WHERE index_name = 'PRIMARY' - AND table_schema = #{scope[:schema]} - AND table_name = #{scope[:name]} - ORDER BY seq_in_index + AND (#{information_schema_scope_sql(table_names)}) + ORDER BY table_name, seq_in_index SQL + + result.each_with_object(Hash.new { |h, k| h[k] = [] }) do |row, primary_keys_by_table| + primary_keys_by_table[row["table_name"]] << row["column_name"] + end end def case_sensitive_comparison(attribute, value) # :nodoc: @@ -1116,32 +1122,62 @@ def configure_connection end def column_definitions(table_name) # :nodoc: - fields = query_all("SHOW FULL FIELDS FROM #{quote_table_name(table_name)}") + fields = column_definitions_for_tables([table_name]).each_value.first + raise ActiveRecord::StatementInvalid.new("Could not find table '#{table_name}'", connection_pool: @pool) unless fields + fields + end - update_fields_for_mariadb(table_name, fields) if mariadb? + def column_definitions_for_tables(table_names) # :nodoc: + return {} if table_names.empty? + + result = query_all(<<~SQL) + SELECT COLUMN_NAME AS 'Field', + COLUMN_TYPE AS 'Type', + COLLATION_NAME AS 'Collation', + IS_NULLABLE AS 'Null', + COLUMN_KEY AS 'Key', + COLUMN_DEFAULT AS 'Default', + EXTRA AS 'Extra', + PRIVILEGES AS 'Privileges', + COLUMN_COMMENT AS 'Comment', + TABLE_NAME AS 'table_name' + FROM information_schema.columns + WHERE #{information_schema_scope_sql(table_names)} + ORDER BY table_name, ORDINAL_POSITION + SQL - fields + fields_by_table = result.each_with_object(Hash.new { |h, k| h[k] = [] }) do |row, hash| + hash[row.delete("table_name")] << row + end + + if mariadb? + fields_by_table.each_value do |fields| + update_fields_for_mariadb(fields) + end + end + + fields_by_table end - def update_fields_for_mariadb(table_name, fields) + def update_fields_for_mariadb(fields) has_function_default_candidate = fields.any? do |field| default = field["Default"] default&.match?(/[a-zA-Z_]\w*\(/) && !/\ACURRENT_TIMESTAMP/i.match?(default) end + return unless has_function_default_candidate + + # MariaDB quotes string literal defaults in information_schema.COLUMNS.COLUMN_DEFAULT + # (e.g. `'uuid()'`) but leaves function/expression defaults unquoted (e.g. `uuid()`), + # unlike SHOW FULL FIELDS which reports both as `uuid()`. Use that to distinguish + # function defaults from literal strings that merely look like function calls, without + # parsing SHOW CREATE TABLE output. + fields.each do |field| + default = field["Default"] + next unless default&.match?(/[a-zA-Z_]\w*\(/) + next if /\ACURRENT_TIMESTAMP/i.match?(default) + next if default.start_with?("'") - if has_function_default_candidate - table_info = create_table_info(table_name) - fields.each do |field| - default = field["Default"] - next unless default&.match?(/[a-zA-Z_]\w*\(/) - next if /\ACURRENT_TIMESTAMP/i.match?(default) - - field_name = field["Field"] - match = table_info&.match(/`#{field_name}` .+ DEFAULT ('|\d+|[A-z]+)/) - if match && match[1].match?(/\A[A-z]/) - field["Extra"] = "DEFAULT_GENERATED" - end - end + field["Extra"] = "DEFAULT_GENERATED" end end diff --git a/activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb b/activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb index 15d2006afb56e..6160de4c72923 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb @@ -6,14 +6,42 @@ module MySQL module SchemaStatements # :nodoc: # Returns an array of indexes for the given table. def indexes(table_name) - indexes = [] + indexes_for_tables([table_name]).each_value.first || [] + end + + def indexes_for_tables(table_names) # :nodoc: + return {} if table_names.empty? + + optional_columns = +"" + optional_columns << ", EXPRESSION AS 'Expression'" if supports_expression_index? + if supports_disabling_indexes? + optional_columns << (mariadb? ? ", IF(ignored = 'NO', 'YES', 'NO') AS 'enabled'" : ", is_visible AS 'enabled'") + end + + result = query_all(<<~SQL) + SELECT TABLE_NAME AS 'Table', INDEX_NAME AS 'Key_name', + NON_UNIQUE AS 'Non_unique', SEQ_IN_INDEX AS 'Seq_in_index', + COLUMN_NAME AS 'Column_name', COLLATION AS 'Collation', + SUB_PART AS 'Sub_part', LOWER(INDEX_TYPE) AS 'Index_type', + INDEX_COMMENT AS 'Index_comment'#{optional_columns} + FROM information_schema.statistics + WHERE (#{information_schema_scope_sql(table_names)}) + AND index_name != 'PRIMARY' + ORDER BY table_name, index_name, seq_in_index + SQL + + indexes_by_table = Hash.new { |h, k| h[k] = [] } + current_index = nil - query_all("SHOW KEYS FROM #{quote_table_name(table_name)}").each do |row| - if current_index != row["Key_name"] - next if row["Key_name"] == "PRIMARY" # skip the primary key - current_index = row["Key_name"] - mysql_index_type = row["Index_type"].downcase.to_sym + result.each do |row| + table = row["Table"] + index_key = [table, row["Key_name"]] + + if current_index != index_key + current_index = index_key + + mysql_index_type = row["Index_type"].to_sym case mysql_index_type when :fulltext, :spatial index_type = mysql_index_type @@ -22,7 +50,7 @@ def indexes(table_name) end index = [ - row["Table"], + table, row["Key_name"], row["Non_unique"].to_i == 0, [], @@ -34,48 +62,57 @@ def indexes(table_name) ] if supports_disabling_indexes? - index[-1][:enabled] = mariadb? ? row["Ignored"] == "NO" : row["Visible"] == "YES" + index[-1][:enabled] = row["enabled"] == "YES" end - indexes << index + indexes_by_table[table] << index end + index = indexes_by_table[table].last + if expression = row["Expression"] expression = expression.gsub("\\'", "'") expression = +"(#{expression})" unless expression.start_with?("(") - indexes.last[-2] << expression - indexes.last[-1][:expressions] ||= {} - indexes.last[-1][:expressions][expression] = expression - indexes.last[-1][:orders][expression] = :desc if row["Collation"] == "D" + index[-2] << expression + index[-1][:expressions] ||= {} + index[-1][:expressions][expression] = expression + index[-1][:orders][expression] = :desc if row["Collation"] == "D" else - indexes.last[-2] << row["Column_name"] - indexes.last[-1][:lengths][row["Column_name"]] = row["Sub_part"].to_i if row["Sub_part"] - indexes.last[-1][:orders][row["Column_name"]] = :desc if row["Collation"] == "D" + index[-2] << row["Column_name"] + index[-1][:lengths][row["Column_name"]] = row["Sub_part"].to_i if row["Sub_part"] + index[-1][:orders][row["Column_name"]] = :desc if row["Collation"] == "D" end end - indexes.map do |index| - options = index.pop + indexes_by_table.transform_values! do |table_indexes| + table_indexes.map! do |index| + options = index.pop - if expressions = options.delete(:expressions) - orders = options.delete(:orders) - lengths = options.delete(:lengths) + if expressions = options.delete(:expressions) + orders = options.delete(:orders) + lengths = options.delete(:lengths) - columns = index[-1].to_h { |name| - [ name.to_sym, expressions[name] || +quote_column_name(name) ] - } + columns = index[-1].to_h { |name| + [ name.to_sym, expressions[name] || +quote_column_name(name) ] + } - index[-1] = add_options_for_index_columns( - columns, order: orders, length: lengths - ).values.join(", ") + index[-1] = add_options_for_index_columns( + columns, order: orders, length: lengths + ).values.join(", ") + end + MySQL::IndexDefinition.new(*index, **options) end - MySQL::IndexDefinition.new(*index, **options) end - rescue StatementInvalid => e - if e.message.match?(/Table '.+' doesn't exist/) - [] - else - raise + + indexes_by_table + end + + def columns_for_tables(table_names) # :nodoc: + definitions_by_table = column_definitions_for_tables(table_names) + definitions_by_table.each_with_object({}) do |(table, definitions), hash| + hash[table] = definitions.map do |field| + new_column_from_field(table, field, definitions) + end end end @@ -197,7 +234,15 @@ def create_table_definition(name, **options) def new_column_from_field(table_name, field, _definitions) type_metadata = fetch_type_metadata(field["Type"], field["Extra"]) default, default_function = field["Default"], nil - + # MariaDB reports the COLUMN_DEFAULT of a nullable column with no + # default as the string "NULL" rather than SQL NULL; normalize it so + # the column is treated as having no default. MySQL already returns + # SQL NULL for a no-default column, and leaves a literal "NULL" + # string default unquoted, so this must be MariaDB-only — otherwise + # MySQL's literal "NULL" defaults would be wiped to nil. A literal + # "NULL" string default on MariaDB is quoted ('NULL') and is handled + # by the strip branch below. + default = nil if mariadb? && default == "NULL" if type_metadata.type == :datetime && /\ACURRENT_TIMESTAMP(?:\([0-6]?\))?\z/i.match?(default) default = "#{default} ON UPDATE #{default}" if /on update CURRENT_TIMESTAMP/i.match?(field["Extra"]) default, default_function = nil, default @@ -209,9 +254,15 @@ def new_column_from_field(table_name, field, _definitions) default = default.gsub("\\'", "'") default, default_function = nil, default end - elsif type_metadata.type == :text && default&.start_with?("'") - # strip and unescape quotes - default = default[1...-1].gsub("\\'", "'") + elsif mariadb? && default&.start_with?("'") + # MariaDB quotes string literal defaults in information_schema + # COLUMN_DEFAULT (e.g. `'abc'`, `'O''Connor'`) and escapes + # embedded single quotes — by doubling them in varchar defaults + # (`''`) but with a backslash in text defaults (`\'`). MySQL leaves + # string defaults unquoted, so this only runs on MariaDB. Strip the + # wrapping quotes and undo both escape forms so the default matches + # what SHOW FULL FIELDS reported. + default = default[1...-1].gsub("''", "'").gsub("\\'", "'") end MySQL::Column.new( @@ -283,6 +334,19 @@ def extract_schema_qualified_name(string) [schema, name] end + def information_schema_scope_sql(table_names) + by_schema = table_names.each_with_object(Hash.new { |h, k| h[k] = [] }) do |name, h| + schema, tbl = extract_schema_qualified_name(name) + h[schema] << tbl + end + + by_schema.map do |schema, names| + schema_sql = schema ? quote(schema) : "database()" + names_sql = names.map { |n| quote(n) }.join(", ") + "(table_schema = #{schema_sql} AND table_name IN (#{names_sql}))" + end.join(" OR ") + end + def type_with_size_to_sql(type, size) case size&.to_s when nil, "tiny", "medium", "long" diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb b/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb index b5a8b114054af..d880f9c374f58 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb @@ -96,10 +96,19 @@ def index_name_exists?(table_name, index_name) # Returns an array of indexes for the given table. def indexes(table_name) # :nodoc: - scope = quoted_scope(table_name) + indexes_for_tables([table_name])[table_name.to_s] + end + + def indexes_for_tables(table_names) # :nodoc: + return {} if table_names.empty? + + result_by_table = table_names.each_with_object({}) do |name, h| + h[name.to_s] = [] + end + key_for = table_key_lookup(table_names) result = query_rows(<<~SQL) - SELECT distinct i.relname, d.indisunique, d.indkey, pg_get_indexdef(d.indexrelid), + SELECT distinct n.nspname, t.relname, i.relname, d.indisunique, d.indkey, pg_get_indexdef(d.indexrelid), pg_catalog.obj_description(i.oid, 'pg_class') AS comment, d.indisvalid, ARRAY( SELECT pg_get_indexdef(d.indexrelid, k + 1, true) @@ -112,60 +121,16 @@ def indexes(table_name) # :nodoc: LEFT JOIN pg_namespace n ON n.oid = t.relnamespace WHERE i.relkind IN ('i', 'I') AND d.indisprimary = 'f' - AND t.relname = #{scope[:name]} - AND n.nspname = #{scope[:schema]} + AND (#{table_scope_sql(table_names)}) ORDER BY i.relname SQL - result.map do |row| - index_name = row[0] - unique = row[1] - indkey = row[2].split(" ").map(&:to_i) - inddef = row[3] - comment = row[4] - valid = row[5] - columns = decode_string_array(row[6]).map { |c| Utils.unquote_identifier(c.strip.gsub('""', '"')) } - - using, expressions, include, nulls_not_distinct, where = inddef.scan(/ USING (\w+?) \((.+?)\)(?: INCLUDE \((.+?)\))?( NULLS NOT DISTINCT)?(?: WHERE (.+))?\z/m).flatten - - orders = {} - opclasses = {} - include_columns = include ? include.split(",").map { |c| Utils.unquote_identifier(c.strip.gsub('""', '"')) } : [] - - if indkey.include?(0) - columns = expressions - else - # prevent INCLUDE columns from being matched - columns.reject! { |c| include_columns.include?(c) } - - # add info on sort order (only desc order is explicitly specified, asc is the default) - # and non-default opclasses - expressions.scan(/(?\w+)"?\s?(?(?:\w+\.)?\w+_ops(_\w+)?)?\s?(?DESC)?\s?(?NULLS (?:FIRST|LAST))?/).each do |column, opclass, desc, nulls| - opclasses[column] = opclass.split(".").last.to_sym if opclass - - if nulls - orders[column] = [desc, nulls].compact.join(" ") - else - orders[column] = :desc if desc - end - end - end - - IndexDefinition.new( - table_name, - index_name, - unique, - columns, - orders: orders, - opclasses: opclasses, - where: where, - using: using.to_sym, - include: include_columns.presence, - nulls_not_distinct: nulls_not_distinct.present?, - comment: comment.presence, - valid: valid - ) + result.each do |row| + key = key_for.call(row[0], row[1]) + result_by_table[key] << index_from_row(row[2..], key) if key end + + result_by_table end def table_options(table_name) # :nodoc: @@ -585,16 +550,43 @@ def pk_and_sequence_for(table) # :nodoc: end def primary_keys(table_name) # :nodoc: - query_values(<<~SQL) - SELECT a.attname + primary_keys_for_tables([table_name])[table_name.to_s] + end + + def primary_keys_for_tables(table_names) # :nodoc: + return {} if table_names.empty? + + result_by_table = table_names.each_with_object({}) do |name, h| + h[name.to_s] = [] + end + key_for = table_key_lookup(table_names) + + result = query_all(<<~SQL) + SELECT n.nspname AS nspname, t.relname AS relname, a.attname AS column_name FROM pg_index i - JOIN pg_attribute a - ON a.attrelid = i.indrelid - AND a.attnum = ANY(i.indkey) - WHERE i.indrelid = #{quote(quote_table_name(table_name))}::regclass - AND i.indisprimary + INNER JOIN pg_class t ON t.oid = i.indrelid + INNER JOIN pg_namespace n ON n.oid = t.relnamespace + INNER JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey) + WHERE i.indisprimary + AND i.indrelid = #{to_regclass_array_sql(table_names)} ORDER BY array_position(i.indkey, a.attnum) SQL + + result.each do |row| + key = key_for.call(row["nspname"], row["relname"]) + result_by_table[key] << row["column_name"] if key + end + + result_by_table + end + + def columns_for_tables(table_names) # :nodoc: + definitions_by_table = column_definitions_for_tables(table_names) + definitions_by_table.each_with_object({}) do |(table, definitions), hash| + hash[table] = definitions.map do |field| + new_column_from_field(table, field, definitions) + end + end end # Renames a table. @@ -1411,6 +1403,109 @@ def extract_schema_qualified_name(string) [name.schema, name.identifier] end + # Build a SQL expression scoping a relation-OID column (e.g. + # +i.indrelid+, +a.attrelid+) to the given table names via + # to_regclass. Each name resolves the way casting to regclass does + # (an unqualified name picks the first schema on the search path; a + # qualified name pins its schema), but to_regclass returns NULL for + # a missing relation instead of raising — so one bad name can't fail + # the whole batch. + def to_regclass_array_sql(table_names) + names_sql = table_names.map { |name| quote(quote_table_name(name)) }.join(", ") + "ANY(ARRAY(SELECT to_regclass(x)::oid FROM unnest(ARRAY[#{names_sql}]) AS x))" + end + + # Returns a proc mapping a row's (namespace, relname) back to the + # requested table-name string, so results are keyed the way + # SchemaCache#add_all looks them up (by +table.to_s+). Qualified + # requests match on (schema, relname); unqualified requests match on + # relname alone (resolved via the search path). + def table_key_lookup(table_names) + qualified = {} + unqualified = {} + table_names.each do |name| + schema, identifier = extract_schema_qualified_name(name) + if schema + qualified[[schema, identifier]] = name.to_s + else + unqualified[identifier] = name.to_s + end + end + ->(nsp, rel) { qualified[[nsp, rel]] || unqualified[rel] } + end + + # Build a WHERE clause scoping pg_class (t) / pg_namespace (n) to the + # given table names, grouped by schema: unqualified names resolve + # against the search path (matching every schema on it that has the + # relation, as +#indexes+ does), qualified names pin their schema. + # Used for relation-name-based queries where matching all search-path + # schemas is the existing behavior. + def table_scope_sql(table_names) + by_schema = table_names.each_with_object(Hash.new { |h, k| h[k] = [] }) do |name, h| + schema, tbl = extract_schema_qualified_name(name) + h[schema] << tbl + end + + by_schema.map do |schema, names| + schema_sql = schema ? quote(schema) : "ANY (current_schemas(false))" + names_sql = names.map { |n| quote(n) }.join(", ") + "(n.nspname = #{schema_sql} AND t.relname IN (#{names_sql}))" + end.join(" OR ") + end + + # Turns one +#indexes_for_tables+ row — the seven columns after the + # leading namespace/relname key columns — into an IndexDefinition, + # attributed to +table_name+ (the requested table-name string). + def index_from_row(row, table_name) + index_name = row[0] + unique = row[1] + indkey = row[2].split(" ").map(&:to_i) + inddef = row[3] + comment = row[4] + valid = row[5] + columns = decode_string_array(row[6]).map { |c| Utils.unquote_identifier(c.strip.gsub('""', '"')) } + + using, expressions, include, nulls_not_distinct, where = inddef.scan(/ USING (\w+?) \((.+?)\)(?: INCLUDE \((.+?)\))?( NULLS NOT DISTINCT)?(?: WHERE (.+))?\z/m).flatten + + orders = {} + opclasses = {} + include_columns = include ? include.split(",").map { |c| Utils.unquote_identifier(c.strip.gsub('""', '"')) } : [] + + if indkey.include?(0) + columns = expressions + else + # prevent INCLUDE columns from being matched + columns.reject! { |c| include_columns.include?(c) } + + # add info on sort order (only desc order is explicitly specified, asc is the default) + # and non-default opclasses + expressions.scan(/(?\w+)"?\s?(?(?:\w+\.)?\w+_ops(_\w+)?)?\s?(?DESC)?\s?(?NULLS (?:FIRST|LAST))?/).each do |column, opclass, desc, nulls| + opclasses[column] = opclass.split(".").last.to_sym if opclass + + if nulls + orders[column] = [desc, nulls].compact.join(" ") + else + orders[column] = :desc if desc + end + end + end + + IndexDefinition.new( + table_name, + index_name, + unique, + columns, + orders: orders, + opclasses: opclasses, + where: where, + using: using.to_sym, + include: include_columns.presence, + nulls_not_distinct: nulls_not_distinct.present?, + comment: comment.presence, + valid: valid + ) + end + def decode_string_array(value) PG::TextDecoder::Array.new.decode(value) end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb b/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb index 2299a6204a238..c4e454f911f38 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb @@ -1259,20 +1259,44 @@ def internal_set_config(setting, value) # - format_type includes the column size constraint, e.g. varchar(50) # - ::regclass is a function that gives the id for a table name def column_definitions(table_name) - query_rows(<<~SQL) - SELECT a.attname, format_type(a.atttypid, a.atttypmod), + fields = column_definitions_for_tables([table_name])[table_name.to_s] + if fields.empty? && query_value("SELECT to_regclass(#{quote(quote_table_name(table_name))})").nil? + raise ActiveRecord::StatementInvalid.new("Could not find table '#{table_name}'", connection_pool: @pool) + end + fields + end + + def column_definitions_for_tables(table_names) # :nodoc: + return {} if table_names.empty? + + result_by_table = table_names.each_with_object({}) do |name, h| + h[name.to_s] = [] + end + key_for = table_key_lookup(table_names) + + result = query_rows(<<~SQL) + SELECT n.nspname, k.relname, a.attname, format_type(a.atttypid, a.atttypmod), pg_get_expr(d.adbin, d.adrelid), a.attnotnull, a.atttypid, a.atttypmod, c.collname, col_description(a.attrelid, a.attnum) AS comment, #{supports_identity_columns? ? 'attidentity' : quote('')} AS identity, #{supports_virtual_columns? ? 'attgenerated' : quote('')} as attgenerated FROM pg_attribute a + INNER JOIN pg_class k ON k.oid = a.attrelid + INNER JOIN pg_namespace n ON n.oid = k.relnamespace LEFT JOIN pg_attrdef d ON a.attrelid = d.adrelid AND a.attnum = d.adnum LEFT JOIN pg_type t ON a.atttypid = t.oid LEFT JOIN pg_collation c ON a.attcollation = c.oid AND a.attcollation <> t.typcollation - WHERE a.attrelid = #{quote(quote_table_name(table_name))}::regclass - AND a.attnum > 0 AND NOT a.attisdropped + WHERE a.attnum > 0 AND NOT a.attisdropped + AND a.attrelid = #{to_regclass_array_sql(table_names)} ORDER BY a.attnum SQL + + result.each do |row| + key = key_for.call(row[0], row[1]) + result_by_table[key] << row[2..] if key + end + + result_by_table end def arel_visitor diff --git a/activerecord/lib/active_record/connection_adapters/schema_cache.rb b/activerecord/lib/active_record/connection_adapters/schema_cache.rb index f1ee10f12bbd4..281bef356c0a0 100644 --- a/activerecord/lib/active_record/connection_adapters/schema_cache.rb +++ b/activerecord/lib/active_record/connection_adapters/schema_cache.rb @@ -390,9 +390,22 @@ def clear_data_source_cache!(_connection, name) end def add_all(pool) # :nodoc: - pool.with_connection do - tables_to_cache(pool).each do |table| - add(pool, table) + pool.with_connection do |connection| + tables = tables_to_cache(pool) + tables.each { |table| @data_sources[deep_deduplicate(table)] = true } + primary_keys_by_table = connection.primary_keys_for_tables(tables) + columns_by_table = connection.columns_for_tables(tables) + indexes_by_table = connection.indexes_for_tables(tables) + tables.each do |table| + table = deep_deduplicate(table) + table_primary_keys = primary_keys_by_table[table.to_s] + # Mirrors #primary_key, unwraps singular keys. + table_primary_keys = table_primary_keys.size > 1 ? table_primary_keys : table_primary_keys.first + @primary_keys[table] = deep_deduplicate(table_primary_keys) + table_columns = deep_deduplicate(columns_by_table[table.to_s]) + @columns[table] = table_columns + @columns_hash[table] = table_columns.index_by(&:name).freeze + @indexes[table] = deep_deduplicate(indexes_by_table[table.to_s]) end version(pool) diff --git a/activerecord/test/cases/adapters/abstract_mysql_adapter/case_insensitive_table_names_test.rb b/activerecord/test/cases/adapters/abstract_mysql_adapter/case_insensitive_table_names_test.rb new file mode 100644 index 0000000000000..956c6f6b8d371 --- /dev/null +++ b/activerecord/test/cases/adapters/abstract_mysql_adapter/case_insensitive_table_names_test.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +require "cases/helper" + +module ActiveRecord + class CaseInsensitiveTableNamesTest < ActiveRecord::AbstractMysqlTestCase + def setup + @connection = ActiveRecord::Base.lease_connection + @connection.drop_table(:MixedCaseTbl, if_exists: true) + end + + def teardown + @connection.drop_table(:MixedCaseTbl, if_exists: true) + end + + # With lower_case_table_names = 1 or 2, table-name comparison is + # case-insensitive, so introspecting "mixedcasetbl" must find the table + # stored as "MixedCaseTbl". The single-table APIs and SchemaCache#add must + # attribute the result to the requested name without depending on the + # database's own (differently-cased) table name. + def test_introspection_with_wrong_case_table_name + skip "only relevant when table name comparison is case-insensitive" if lower_case_table_names.to_i == 0 + + @connection.create_table(:MixedCaseTbl, force: true) do |t| + t.string :name + t.integer :custom_id + end + @connection.add_index(:MixedCaseTbl, :name) + + assert_equal %w[id name custom_id], @connection.columns("mixedcasetbl").map(&:name) + assert_equal %w[id], @connection.primary_keys("mixedcasetbl") + assert_equal 1, @connection.indexes("mixedcasetbl").size + + cache = ActiveRecord::Base.connection_pool.schema_cache + cache.clear_data_source_cache!("mixedcasetbl") + cache.add("mixedcasetbl") + + assert_equal %w[id name custom_id], cache.columns("mixedcasetbl").map(&:name) + assert_equal "id", cache.primary_keys("mixedcasetbl") + assert_equal 1, cache.indexes("mixedcasetbl").size + end + + private + def lower_case_table_names + @connection.show_variable("lower_case_table_names") + end + end +end diff --git a/activerecord/test/cases/adapters/abstract_mysql_adapter/schema_test.rb b/activerecord/test/cases/adapters/abstract_mysql_adapter/schema_test.rb index 4fd188f3ecc74..9006384280959 100644 --- a/activerecord/test/cases/adapters/abstract_mysql_adapter/schema_test.rb +++ b/activerecord/test/cases/adapters/abstract_mysql_adapter/schema_test.rb @@ -92,6 +92,136 @@ def test_dump_indexes assert_equal :fulltext, index_c.type end + def test_indexes_for_multiple_tables + @connection.create_table(:idx_multi_a) { |t| t.string :email; t.string :name } + @connection.create_table(:idx_multi_b) { |t| t.string :email; t.string :other } + # "by_email" is idx_multi_a's alphabetically-last index and + # idx_multi_b's alphabetically-first, so the rows are adjacent across + # the two tables in the ORDER BY. A name-only index boundary would fail + # to start a new index for idx_multi_b's "by_email" and crash. + @connection.add_index :idx_multi_a, :name, name: "aaa_name" + @connection.add_index :idx_multi_a, :email, name: "by_email" + @connection.add_index :idx_multi_b, :email, name: "by_email" + @connection.add_index :idx_multi_b, :other, name: "zzz_other" + + # A single table name returns an Array of indexes (backward compatible). + single = @connection.indexes("idx_multi_a") + assert_kind_of Array, single + assert_equal %w[aaa_name by_email], single.map(&:name).sort + + # indexes_for_tables returns a Hash of table name => Array of indexes. + multi = @connection.indexes_for_tables(["idx_multi_a", "idx_multi_b"]) + assert_kind_of Hash, multi + assert_equal %w[idx_multi_a idx_multi_b], multi.keys.sort + assert multi.values.all?(Array) + assert_equal %w[aaa_name by_email], multi["idx_multi_a"].map(&:name).sort + assert_equal %w[by_email zzz_other], multi["idx_multi_b"].map(&:name).sort + + # A non-primary index name shared across two tables must not merge: + # each table's "by_email" keeps its own columns and table attribute. + a = multi["idx_multi_a"].find { |i| i.name == "by_email" } + b = multi["idx_multi_b"].find { |i| i.name == "by_email" } + assert_equal %w[email], a.columns + assert_equal %w[email], b.columns + assert_equal "idx_multi_a", a.table + assert_equal "idx_multi_b", b.table + ensure + @connection.drop_table :idx_multi_a, if_exists: true + @connection.drop_table :idx_multi_b, if_exists: true + end + + def test_indexes_for_multiple_tables_with_qualified_and_unqualified_names + @connection.create_table(:idx_mix_a) { |t| t.string :name } + @connection.create_table("#{@db_name}.idx_mix_b") { |t| t.string :other } + @connection.add_index :idx_mix_a, :name, name: "mix_a_name" + @connection.add_index "#{@db_name}.idx_mix_b", :other, name: "mix_b_other" + + # Mixing a schema-qualified name with an unqualified one yields a + # multi-clause (OR) scope. The trailing AND index_name != 'PRIMARY' + # must filter every clause; without parenthesizing the scope it binds + # only to the last clause and the primary key leaks in as a "PRIMARY" + # index for the other table. + multi = @connection.indexes_for_tables(["#{@db_name}.idx_mix_b", :idx_mix_a]) + assert_equal %w[mix_b_other], multi["idx_mix_b"].map(&:name).sort + assert_equal %w[mix_a_name], multi["idx_mix_a"].map(&:name).sort + ensure + @connection.drop_table :idx_mix_a, if_exists: true + @connection.drop_table "#{@db_name}.idx_mix_b", if_exists: true + end + + def test_columns_for_multiple_tables + @connection.create_table(:cols_multi_a) { |t| t.string :title; t.integer :counter } + @connection.create_table(:cols_multi_b) { |t| t.string :name; t.text :body } + + # A single table name returns an Array of columns (backward compatible). + assert_kind_of Array, @connection.columns("cols_multi_a") + + # columns_for_tables returns a Hash of table name => Array of columns, + # matching #columns for each table (names and sql_types). + multi = @connection.columns_for_tables(["cols_multi_a", "cols_multi_b"]) + assert_kind_of Hash, multi + assert_equal %w[cols_multi_a cols_multi_b], multi.keys.sort + + profile = ->(cols) { cols.map { |c| [c.name, c.sql_type] } } + assert_equal profile.call(@connection.columns("cols_multi_a")), profile.call(multi["cols_multi_a"]) + assert_equal profile.call(@connection.columns("cols_multi_b")), profile.call(multi["cols_multi_b"]) + ensure + @connection.drop_table :cols_multi_a, if_exists: true + @connection.drop_table :cols_multi_b, if_exists: true + end + + def test_primary_keys_for_multiple_tables + @connection.create_table(:pk_multi_a, primary_key: :custom_id) { |t| t.string :name } + @connection.create_table(:pk_multi_b) { |t| t.string :name } + @connection.execute "CREATE TABLE pk_multi_c (a int NOT NULL, b int NOT NULL, c int NOT NULL, PRIMARY KEY (c, a, b))" + @connection.create_table(:pk_multi_none, id: false) { |t| t.string :name } + + # A single table name returns an Array of primary key columns (backward compatible). + assert_equal %w[custom_id], @connection.primary_keys("pk_multi_a") + assert_equal %w[id], @connection.primary_keys("pk_multi_b") + # Composite key columns arrive in PRIMARY KEY ordinal order (seq_in_index), + # not the table's column order. + assert_equal %w[c a b], @connection.primary_keys("pk_multi_c") + # A table without a primary key returns an empty Array, not a missing key. + assert_equal [], @connection.primary_keys("pk_multi_none") + + # primary_keys_for_tables returns a Hash of table name => Array of pk columns. + multi = @connection.primary_keys_for_tables(["pk_multi_a", "pk_multi_b", "pk_multi_c", "pk_multi_none"]) + assert_kind_of Hash, multi + assert_equal %w[pk_multi_a pk_multi_b pk_multi_c], multi.keys.sort + assert_equal %w[custom_id], multi["pk_multi_a"] + assert_equal %w[id], multi["pk_multi_b"] + assert_equal %w[c a b], multi["pk_multi_c"] + # No primary key => absent from the hash; the default-proc returns [] on + # access (so the cache reduces it to nil, as #primary_key does). + assert_not multi.key?("pk_multi_none") + assert_equal [], multi["pk_multi_none"] + ensure + @connection.drop_table :pk_multi_a, if_exists: true + @connection.drop_table :pk_multi_b, if_exists: true + @connection.drop_table :pk_multi_c, if_exists: true + @connection.drop_table :pk_multi_none, if_exists: true + end + + def test_primary_keys_for_multiple_tables_with_qualified_and_unqualified_names + @connection.create_table(:pk_mix_a) { |t| t.string :name } + @connection.create_table("#{@db_name}.pk_mix_b") { |t| t.string :other } + @connection.add_index :pk_mix_a, :name, name: "pk_mix_a_name" + @connection.add_index "#{@db_name}.pk_mix_b", :other, name: "pk_mix_b_other" + + # Mixing a schema-qualified name with an unqualified one yields a + # multi-clause (OR) scope. `index_name = 'PRIMARY' AND ` must + # filter every clause; without parenthesizing the scope, AND binds + # only to the first clause and the other table's non-primary indexes + # leak in as primary key columns. + multi = @connection.primary_keys_for_tables(["#{@db_name}.pk_mix_b", :pk_mix_a]) + assert_equal %w[id], multi["pk_mix_b"] + assert_equal %w[id], multi["pk_mix_a"] + ensure + @connection.drop_table :pk_mix_a, if_exists: true + @connection.drop_table "#{@db_name}.pk_mix_b", if_exists: true + end + unless mysql_enforcing_gtid_consistency? def test_drop_temporary_table @connection.transaction do diff --git a/activerecord/test/cases/adapters/postgresql/schema_test.rb b/activerecord/test/cases/adapters/postgresql/schema_test.rb index 844150ec451dc..f3298a7e54073 100644 --- a/activerecord/test/cases/adapters/postgresql/schema_test.rb +++ b/activerecord/test/cases/adapters/postgresql/schema_test.rb @@ -291,6 +291,111 @@ def test_data_source_exists_quoted_table end end + def test_primary_keys_for_multiple_tables + @connection.create_table(:pg_pk_multi_a, primary_key: :custom_id) { |t| t.string :name } + @connection.create_table(:pg_pk_multi_b) { |t| t.string :name } + @connection.execute "CREATE TABLE pg_pk_multi_c (a int NOT NULL, b int NOT NULL, c int NOT NULL, PRIMARY KEY (c, a, b))" + @connection.create_table(:pg_pk_multi_none, id: false) { |t| t.string :name } + + # A single table name returns an Array of primary key columns (backward compatible). + assert_equal %w[custom_id], @connection.primary_keys("pg_pk_multi_a") + assert_equal %w[id], @connection.primary_keys("pg_pk_multi_b") + # Composite key columns arrive in PRIMARY KEY ordinal order, not column order. + assert_equal %w[c a b], @connection.primary_keys("pg_pk_multi_c") + # A table without a primary key returns an empty Array, not a missing key. + assert_equal [], @connection.primary_keys("pg_pk_multi_none") + + # primary_keys_for_tables returns a Hash keyed by the requested table name. + multi = @connection.primary_keys_for_tables([ + "pg_pk_multi_a", "pg_pk_multi_b", "pg_pk_multi_c", "pg_pk_multi_none", + "#{SCHEMA_NAME}.#{PK_TABLE_NAME}", + ]) + assert_kind_of Hash, multi + assert_equal %w[pg_pk_multi_a pg_pk_multi_b pg_pk_multi_c pg_pk_multi_none test_schema.table_with_pk], multi.keys.sort + assert_equal %w[custom_id], multi["pg_pk_multi_a"] + assert_equal %w[id], multi["pg_pk_multi_b"] + assert_equal %w[c a b], multi["pg_pk_multi_c"] + assert_equal [], multi["pg_pk_multi_none"] + # A schema-qualified request is keyed by the requested string, not the relname. + assert_equal %w[id], multi["#{SCHEMA_NAME}.#{PK_TABLE_NAME}"] + ensure + @connection.drop_table :pg_pk_multi_a, if_exists: true + @connection.drop_table :pg_pk_multi_b, if_exists: true + @connection.drop_table :pg_pk_multi_c, if_exists: true + @connection.drop_table :pg_pk_multi_none, if_exists: true + end + + def test_columns_for_multiple_tables + @connection.create_table(:pg_cols_multi_a) { |t| t.string :title; t.integer :counter } + @connection.create_table(:pg_cols_multi_b) { |t| t.string :name; t.text :body } + + # A single table name returns an Array of columns (backward compatible). + assert_kind_of Array, @connection.columns("pg_cols_multi_a") + + # columns_for_tables returns a Hash of table name => Array of columns, + # matching #columns for each table (names and sql_types). + multi = @connection.columns_for_tables(["pg_cols_multi_a", "pg_cols_multi_b", "#{SCHEMA_NAME}.#{TABLE_NAME}"]) + assert_kind_of Hash, multi + assert_equal %w[pg_cols_multi_a pg_cols_multi_b test_schema.things], multi.keys.sort + assert_equal 6, multi["#{SCHEMA_NAME}.#{TABLE_NAME}"].size + + profile = ->(cols) { cols.map { |c| [c.name, c.sql_type] } } + assert_equal profile.call(@connection.columns("pg_cols_multi_a")), profile.call(multi["pg_cols_multi_a"]) + assert_equal profile.call(@connection.columns("pg_cols_multi_b")), profile.call(multi["pg_cols_multi_b"]) + # A schema-qualified request is keyed by the requested string and matches #columns. + assert_equal profile.call(@connection.columns("#{SCHEMA_NAME}.#{TABLE_NAME}")), profile.call(multi["#{SCHEMA_NAME}.#{TABLE_NAME}"]) + ensure + @connection.drop_table :pg_cols_multi_a, if_exists: true + @connection.drop_table :pg_cols_multi_b, if_exists: true + end + + def test_columns_for_zero_column_table + @connection.execute 'CREATE TABLE "pg_zero_col" ()' + assert_equal [], @connection.columns("pg_zero_col") + ensure + @connection.drop_table :pg_zero_col, if_exists: true + end + + def test_columns_raises_for_missing_table + assert_raises(ActiveRecord::StatementInvalid) do + @connection.columns("pg_does_not_exist") + end + end + + def test_indexes_for_multiple_tables + @connection.create_table(:pg_idx_multi_a) { |t| t.string :email; t.string :name } + @connection.create_table(:pg_idx_multi_b) { |t| t.string :email; t.string :other } + @connection.add_index :pg_idx_multi_a, :name, name: "idx_a_name" + @connection.add_index :pg_idx_multi_a, :email, name: "idx_a_email" + @connection.add_index :pg_idx_multi_b, :email, name: "idx_b_email" + @connection.add_index :pg_idx_multi_b, :other, name: "idx_b_other" + + # A single table name returns an Array of indexes (backward compatible). + single = @connection.indexes("pg_idx_multi_a") + assert_kind_of Array, single + assert_equal %w[idx_a_email idx_a_name], single.map(&:name).sort + + # indexes_for_tables returns a Hash of table name => Array of indexes. + multi = @connection.indexes_for_tables(["pg_idx_multi_a", "pg_idx_multi_b", "#{SCHEMA_NAME}.#{TABLE_NAME}"]) + assert_kind_of Hash, multi + assert_equal %w[pg_idx_multi_a pg_idx_multi_b test_schema.things], multi.keys.sort + assert multi.values.all?(Array) + assert_equal %w[idx_a_email idx_a_name], multi["pg_idx_multi_a"].map(&:name).sort + assert_equal %w[idx_b_email idx_b_other], multi["pg_idx_multi_b"].map(&:name).sort + + # Each index is attributed to its requested table name. + a = multi["pg_idx_multi_a"].find { |i| i.name == "idx_a_email" } + assert_equal %w[email], a.columns + assert_equal "pg_idx_multi_a", a.table + + # A schema-qualified request is keyed by the requested string and matches #indexes. + assert_equal @connection.indexes("#{SCHEMA_NAME}.#{TABLE_NAME}").map(&:name).sort, + multi["#{SCHEMA_NAME}.#{TABLE_NAME}"].map(&:name).sort + ensure + @connection.drop_table :pg_idx_multi_a, if_exists: true + @connection.drop_table :pg_idx_multi_b, if_exists: true + end + def test_with_schema_prefixed_table_name assert_nothing_raised do assert_equal COLUMNS, columns("#{SCHEMA_NAME}.#{TABLE_NAME}") diff --git a/activerecord/test/cases/defaults_test.rb b/activerecord/test/cases/defaults_test.rb index cc730c66c3daf..b16da086bc05c 100644 --- a/activerecord/test/cases/defaults_test.rb +++ b/activerecord/test/cases/defaults_test.rb @@ -182,6 +182,28 @@ class MysqlDefaultExpressionTest < ActiveRecord::TestCase end end + test "literal NULL string default is not mistaken for no default" do + connection = ActiveRecord::Base.lease_connection + connection.create_table :default_null_string, force: true do |t| + t.string :name, default: "NULL" + end + column = connection.columns(:default_null_string).find { |c| c.name == "name" } + assert_equal "NULL", column.default + ensure + connection&.drop_table :default_null_string, if_exists: true + end + + test "string default containing a single quote is unescaped correctly" do + connection = ActiveRecord::Base.lease_connection + connection.create_table :default_quote_string, force: true do |t| + t.string :name, default: "O'Connor" + end + column = connection.columns(:default_quote_string).find { |c| c.name == "name" } + assert_equal "O'Connor", column.default + ensure + connection&.drop_table :default_quote_string, if_exists: true + end + test "schema dump datetime includes default expression" do output = dump_table_schema("datetime_defaults") assert_match %r/t\.datetime\s+"modified_datetime",\s+precision: nil,\s+default: -> { "CURRENT_TIMESTAMP(?:\(\))?" }/i, output diff --git a/activerecord/test/cases/migration_test.rb b/activerecord/test/cases/migration_test.rb index 1ea97f99b08df..be00635f908e9 100644 --- a/activerecord/test/cases/migration_test.rb +++ b/activerecord/test/cases/migration_test.rb @@ -1542,6 +1542,17 @@ def test_default_functions_on_columns end end + if current_adapter?(:Mysql2Adapter, :TrilogyAdapter) + def test_literal_default_that_looks_like_a_function_is_not_mistaken_for_one + with_bulk_change_table do |t| + t.string :name, default: "uuid()" + end + + assert_equal "uuid()", column(:name).default + assert_nil column(:name).default_function + end + end + if current_adapter?(:Mysql2Adapter, :TrilogyAdapter) def test_updating_auto_increment with_bulk_change_table do |t|