feat(lora): Add FIM-guided adaptive LoRA rank allocation (FimConfig + initialize_lora_fim_ranks)

src/peft/__init__.py

@@ -65,6 +65,7 @@
     DeloraConfig,
     DeloraModel,
     EvaConfig,
+    FimConfig,
     FourierFTConfig,
     FourierFTModel,
     GraloraConfig,
@@ -132,6 +133,7 @@
     create_arrow_model,
     get_eva_state_dict,
     initialize_lora_eva_weights,
+    initialize_lora_fim_ranks,
     preprocess_loraga,
     save_as_lora,
 )
@@ -189,6 +191,7 @@
     "DeloraConfig",
     "DeloraModel",
     "EvaConfig",
+    "FimConfig",
     "FourierFTConfig",
     "FourierFTModel",
     "GraloraConfig",
@@ -280,6 +283,7 @@
     "initialize_kv_prefix_from_past_key_values",
     "initialize_kv_prefix_from_text",
     "initialize_lora_eva_weights",
+    "initialize_lora_fim_ranks",
     "inject_adapter_in_model",
     "load_peft_weights",
     "prepare_model_for_kbit_training",

src/peft/tuners/__init__.py

@@ -32,6 +32,7 @@
     ArrowConfig,
     BdLoraConfig,
     EvaConfig,
+    FimConfig,
     LoftQConfig,
     LoraConfig,
     LoraGAConfig,
@@ -41,6 +42,7 @@
     create_arrow_model,
     get_eva_state_dict,
     initialize_lora_eva_weights,
+    initialize_lora_fim_ranks,
     preprocess_loraga,
     save_as_lora,
 )
@@ -87,6 +89,7 @@
     "DeloraConfig",
     "DeloraModel",
     "EvaConfig",
+    "FimConfig",
     "FourierFTConfig",
     "FourierFTModel",
     "GraloraConfig",
@@ -156,6 +159,7 @@
     "create_arrow_model",
     "get_eva_state_dict",
     "initialize_lora_eva_weights",
+    "initialize_lora_fim_ranks",
     "preprocess_loraga",
     "save_as_lora",
 ]

src/peft/tuners/lora/__init__.py

@@ -19,6 +19,7 @@
 from .config import ArrowConfig, BdLoraConfig, EvaConfig, LoftQConfig, LoraConfig, LoraGAConfig, LoraRuntimeConfig
 from .conversion import convert_to_lora, save_as_lora
 from .eva import get_eva_state_dict, initialize_lora_eva_weights
+from .fim import FimConfig, initialize_lora_fim_ranks
 from .gptq import GPTQLoraLinear
 from .layer import Conv2d, Conv3d, Embedding, Linear, LoraLayer, ParamWrapper
 from .loraga import preprocess_loraga
@@ -32,6 +33,7 @@
     "Conv3d",
     "Embedding",
     "EvaConfig",
+    "FimConfig",
     "GPTQLoraLinear",
     "Linear",
     "LoftQConfig",
@@ -45,6 +47,7 @@
     "create_arrow_model",
     "get_eva_state_dict",
     "initialize_lora_eva_weights",
+    "initialize_lora_fim_ranks",
     "preprocess_loraga",
     "save_as_lora",
 ]

src/peft/tuners/lora/config.py

@@ -512,7 +512,17 @@ class LoraConfig(PeftConfig):
     )
     init_lora_weights: (
         bool
-        | Literal["gaussian", "eva", "olora", "pissa", "pissa_niter_[number of iters]", "corda", "loftq", "orthogonal"]
+        | Literal[
+            "gaussian",
+            "eva",
+            "fim",
+            "olora",
+            "pissa",
+            "pissa_niter_[number of iters]",
+            "corda",
+            "loftq",
+            "orthogonal",
+        ]
     ) = field(
         default=True,
         metadata={
@@ -628,6 +638,16 @@ class LoraConfig(PeftConfig):
             )
         },
     )
+    fim_config: Optional["FimConfig"] = field(
+        default=None,
+        metadata={
+            "help": (
+                "The configuration for FIM-guided adaptive rank allocation. If this is passed, then a calibration "
+                "forward+backward pass will redistribute LoRA ranks according to per-layer Fisher Information Matrix "
+                "scores. Also set `init_lora_weights='fim'` and call `initialize_lora_fim_ranks` after wrapping."
+            )
+        },
+    )
     corda_config: Optional[CordaConfig] = field(
         default=None,
         metadata={
@@ -830,6 +850,14 @@ def __post_init__(self):
         elif self.init_lora_weights != "eva" and self.eva_config is not None:
             warnings.warn("`eva_config` specified but will be ignored when `init_lora_weights` is not 'eva'.")
 
+        elif self.init_lora_weights == "fim" and self.fim_config is None:
+            warnings.warn("`init_lora_weights` is 'fim' but `fim_config` is not specified. Using default FimConfig.")
+            from .fim import FimConfig
+
+            self.fim_config = FimConfig()
+        elif self.init_lora_weights != "fim" and self.fim_config is not None:
+            warnings.warn("`fim_config` specified but will be ignored when `init_lora_weights` is not 'fim'.")
+
         elif self.init_lora_weights == "corda" and self.corda_config is None:
             warnings.warn(
                 "`init_lora_weights` is 'corda' but `corda_config` is not specified. Using default CorDA config."