diff --git a/QEfficient/transformers/models/falcon/modeling_falcon.py b/QEfficient/transformers/models/falcon/modeling_falcon.py index 1cfdf88e1..63c80d501 100644 --- a/QEfficient/transformers/models/falcon/modeling_falcon.py +++ b/QEfficient/transformers/models/falcon/modeling_falcon.py @@ -8,9 +8,10 @@ """PyTorch Falcon model.""" import math -from typing import Optional, Tuple, Union +from typing import Optional, Tuple, Type, Union import torch +import torch.nn as nn import torch.utils.checkpoint from torch.nn import functional as F from transformers.cache_utils import Cache @@ -353,6 +354,16 @@ class QEffFalconForCausalLM(FalconForCausalLM): - update the hidden_states, and fix for onnx model """ + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {QEffFalconDecoderLayer} + def forward( self, input_ids: torch.LongTensor = None, diff --git a/QEfficient/transformers/models/gemma/modeling_gemma.py b/QEfficient/transformers/models/gemma/modeling_gemma.py index 1edb8ef53..ee4671ca9 100644 --- a/QEfficient/transformers/models/gemma/modeling_gemma.py +++ b/QEfficient/transformers/models/gemma/modeling_gemma.py @@ -5,7 +5,7 @@ # # ----------------------------------------------------------------------------- -from typing import List, Optional, Tuple, Union +from typing import List, Optional, Tuple, Type, Union import torch from torch import nn @@ -336,6 +336,16 @@ class QEffGemmaForCausalLM(GemmaForCausalLM): - add new args cache idx for the kv retention """ + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {QEffGemmaDecoderLayer} + def forward( self, input_ids: torch.LongTensor = None, diff --git a/QEfficient/transformers/models/gemma2/modeling_gemma2.py b/QEfficient/transformers/models/gemma2/modeling_gemma2.py index 2944601c9..ce3e1cd99 100644 --- a/QEfficient/transformers/models/gemma2/modeling_gemma2.py +++ b/QEfficient/transformers/models/gemma2/modeling_gemma2.py @@ -5,7 +5,7 @@ # # ----------------------------------------------------------------------------- -from typing import Callable, List, Optional, Tuple, Union +from typing import Callable, List, Optional, Tuple, Type, Union import torch from torch import nn @@ -388,6 +388,16 @@ class QEffGemma2ForCausalLM(Gemma2ForCausalLM, GenerationMixin): - add new args cache idx for the kv retention """ + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {QEffGemma2DecoderLayer} + def forward( self, input_ids: torch.LongTensor = None, diff --git a/QEfficient/transformers/models/gemma3/modeling_gemma3.py b/QEfficient/transformers/models/gemma3/modeling_gemma3.py index c91d2fe32..62486837d 100644 --- a/QEfficient/transformers/models/gemma3/modeling_gemma3.py +++ b/QEfficient/transformers/models/gemma3/modeling_gemma3.py @@ -6,7 +6,7 @@ # ----------------------------------------------------------------------------- import copy -from typing import List, Optional, Tuple, Union +from typing import List, Optional, Tuple, Type, Union import torch from torch import nn @@ -589,6 +589,16 @@ def __init__(self, model): self.model = model self.model.vision_model = self.model.vision_tower + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {self.model.vision_tower.vision_model.encoder.layers[0].__class__} + def forward(self, pixel_values): image_features = self.model.get_image_features(pixel_values=pixel_values) return image_features @@ -602,6 +612,16 @@ def __init__(self, model): self.config = self.model.config self.lm_head = self.model.lm_head + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {QEffGemma3DecoderLayer} + def forward( self, input_ids, diff --git a/QEfficient/transformers/models/gpt2/modeling_gpt2.py b/QEfficient/transformers/models/gpt2/modeling_gpt2.py index 6136a2c5d..60424c851 100644 --- a/QEfficient/transformers/models/gpt2/modeling_gpt2.py +++ b/QEfficient/transformers/models/gpt2/modeling_gpt2.py @@ -5,7 +5,7 @@ # # ----------------------------------------------------------------------------- -from typing import Callable, Optional, Tuple, Union +from typing import Callable, Optional, Tuple, Type, Union import torch from torch import nn @@ -397,6 +397,16 @@ class QEffGPT2LMHeadModel(GPT2LMHeadModel): - add new args position idx for the cache_kwargs for kv retention """ + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {QEffGPT2Block} + def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/QEfficient/transformers/models/gpt_bigcode/modeling_gpt_bigcode.py b/QEfficient/transformers/models/gpt_bigcode/modeling_gpt_bigcode.py index 85ea42674..0354ec34c 100644 --- a/QEfficient/transformers/models/gpt_bigcode/modeling_gpt_bigcode.py +++ b/QEfficient/transformers/models/gpt_bigcode/modeling_gpt_bigcode.py @@ -7,7 +7,7 @@ """PyTorch GPTBigCode model.""" -from typing import Optional, Tuple, Union +from typing import Optional, Tuple, Type, Union import torch import torch.utils.checkpoint @@ -378,6 +378,16 @@ def forward( class QEffGPTBigCodeForCausalLM(GPTBigCodeForCausalLM): + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {QEffGPTBigCodeBlock} + def forward( self, input_ids: Optional[torch.Tensor] = None, diff --git a/QEfficient/transformers/models/gpt_oss/modeling_gpt_oss.py b/QEfficient/transformers/models/gpt_oss/modeling_gpt_oss.py index 3efe890b8..b82cd7c81 100644 --- a/QEfficient/transformers/models/gpt_oss/modeling_gpt_oss.py +++ b/QEfficient/transformers/models/gpt_oss/modeling_gpt_oss.py @@ -6,7 +6,7 @@ # ----------------------------------------------------------------------------- import math import os -from typing import Callable, Optional, Union +from typing import Callable, Optional, Type, Union import torch from torch import nn @@ -1205,6 +1205,16 @@ def forward( class QEffGptOssForCausalLM(GptOssForCausalLM): + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {QEffGptOssDecoderLayer} + def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/QEfficient/transformers/models/gptj/modeling_gptj.py b/QEfficient/transformers/models/gptj/modeling_gptj.py index 1a9e45e97..3fad774ef 100644 --- a/QEfficient/transformers/models/gptj/modeling_gptj.py +++ b/QEfficient/transformers/models/gptj/modeling_gptj.py @@ -7,7 +7,7 @@ """PyTorch GPT-J model.""" -from typing import Optional, Tuple, Union +from typing import Optional, Tuple, Type, Union import torch from torch import nn @@ -318,6 +318,16 @@ class QEffGPTJForCausalLM(GPTJForCausalLM): - update the hidden_states, and fix for onnx model """ + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {QEffGPTJBlock} + def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/QEfficient/transformers/models/granite/modeling_granite.py b/QEfficient/transformers/models/granite/modeling_granite.py index 62be5f54d..f15b6b9af 100644 --- a/QEfficient/transformers/models/granite/modeling_granite.py +++ b/QEfficient/transformers/models/granite/modeling_granite.py @@ -5,7 +5,7 @@ # # ----------------------------------------------------------------------------- -from typing import Callable, List, Optional, Tuple, Union +from typing import Callable, List, Optional, Tuple, Type, Union import torch from torch import nn @@ -347,6 +347,16 @@ class QEffGraniteForCausalLM(GraniteForCausalLM): Copied from GraniteForCausalLM: https://github.com/huggingface/transformers/blob/main/src/transformers/models/granite/modeling_granite.py """ + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {QEffGraniteDecoderLayer} + def forward( self, input_ids: torch.LongTensor = None, diff --git a/QEfficient/transformers/models/granitemoe/modeling_granitemoe.py b/QEfficient/transformers/models/granitemoe/modeling_granitemoe.py index b158b4046..7147ab667 100644 --- a/QEfficient/transformers/models/granitemoe/modeling_granitemoe.py +++ b/QEfficient/transformers/models/granitemoe/modeling_granitemoe.py @@ -5,7 +5,7 @@ # # ----------------------------------------------------------------------------- -from typing import List, Optional, Tuple, Union +from typing import List, Optional, Tuple, Type, Union import torch import torch.nn.functional as F @@ -493,6 +493,16 @@ class QEffGraniteMoeForCausalLM(GraniteMoeForCausalLM): Copied from GraniteForCausalLM: https://github.com/huggingface/transformers/blob/main/src/transformers/models/granite/modeling_granite.py """ + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {self.model.layers[0].__class__} + def forward( self, input_ids: torch.LongTensor = None, diff --git a/QEfficient/transformers/models/grok_1/modeling_grok1.py b/QEfficient/transformers/models/grok_1/modeling_grok1.py index 2d8fc412d..a930f2383 100644 --- a/QEfficient/transformers/models/grok_1/modeling_grok1.py +++ b/QEfficient/transformers/models/grok_1/modeling_grok1.py @@ -5,7 +5,7 @@ # # ---------------------------------------------------------------------------- -from typing import List, Optional, Tuple, Union +from typing import List, Optional, Tuple, Type, Union import torch import torch.nn as nn @@ -397,6 +397,16 @@ class QEffGrok1ModelForCausalLM(nn.Module): Grok model for causal language modeling. """ + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {QEffGrok1DecoderLayer} + def forward( self, input_ids: torch.LongTensor = None, diff --git a/QEfficient/transformers/models/internvl/modeling_internvl.py b/QEfficient/transformers/models/internvl/modeling_internvl.py index 85c331aa8..0199739a7 100644 --- a/QEfficient/transformers/models/internvl/modeling_internvl.py +++ b/QEfficient/transformers/models/internvl/modeling_internvl.py @@ -5,7 +5,7 @@ # # ----------------------------------------------------------------------------- -from typing import List, Optional +from typing import List, Optional, Type import torch import torch.nn as nn @@ -21,6 +21,16 @@ def __init__(self, model): super().__init__() self.model = model + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {self.model.vision_model.encoder.layers[0].__class__} + def forward(self, pixel_values): vision_embeds = self.model.extract_feature(pixel_values) # Reshape from [num_patches, 256, hidden_dim] -> [1, num_patches*256, head_dim] @@ -36,6 +46,16 @@ def __init__(self, model): self.config = self.model.language_model.config self.language_model = self.model.language_model + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {self.model.language_model.model.layers[0].__class__} + def forward( self, input_ids, diff --git a/QEfficient/transformers/models/llama/modeling_llama.py b/QEfficient/transformers/models/llama/modeling_llama.py index fb3aed556..56dbaaa16 100644 --- a/QEfficient/transformers/models/llama/modeling_llama.py +++ b/QEfficient/transformers/models/llama/modeling_llama.py @@ -5,7 +5,7 @@ # # ----------------------------------------------------------------------------- -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Type, Union import torch from torch import nn @@ -404,6 +404,16 @@ class QEffLlamaForCausalLM(LlamaForCausalLM): Copied from LlamaForCausalLM: https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py """ + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {QEffLlamaDecoderLayer} + def forward( self, input_ids: torch.LongTensor = None, diff --git a/QEfficient/transformers/models/llama4/modeling_llama4.py b/QEfficient/transformers/models/llama4/modeling_llama4.py index 7a2f687fe..5ee1c8a53 100644 --- a/QEfficient/transformers/models/llama4/modeling_llama4.py +++ b/QEfficient/transformers/models/llama4/modeling_llama4.py @@ -6,7 +6,7 @@ # ----------------------------------------------------------------------------- import math -from typing import Callable, List, Optional, Tuple, Union +from typing import Callable, List, Optional, Tuple, Type, Union import torch from torch import nn @@ -822,6 +822,16 @@ def __init__(self, model): super().__init__() self.model = model + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {self.model.vision_model.model.layers[0].__class__} + def forward(self, pixel_values): vision_feature_layer = self.model.config.vision_config.vision_feature_layer vision_feature_select_strategy = self.model.config.vision_config.vision_feature_select_strategy @@ -849,6 +859,16 @@ def __init__(self, model): self.language_model = self.model.language_model self.config = self.model.config + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {QEffLlama4TextDecoderLayer} + def forward( self, input_ids, diff --git a/QEfficient/transformers/models/llama_swiftkv/modeling_llama_swiftkv.py b/QEfficient/transformers/models/llama_swiftkv/modeling_llama_swiftkv.py index fa42b3f96..67879a894 100644 --- a/QEfficient/transformers/models/llama_swiftkv/modeling_llama_swiftkv.py +++ b/QEfficient/transformers/models/llama_swiftkv/modeling_llama_swiftkv.py @@ -11,7 +11,7 @@ """Inference-only LLaMA model compatible with HuggingFace weights.""" import math -from typing import List, Optional, Tuple, Union +from typing import List, Optional, Tuple, Type, Union import torch from torch import nn @@ -416,6 +416,16 @@ def __init__(self, config: QEffLlamaSwiftKVConfig): self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) self.config = config + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {QEffLlamaSwiftKVDecoderLayer} + def forward( self, input_ids: torch.Tensor, diff --git a/QEfficient/transformers/models/llava/modeling_llava.py b/QEfficient/transformers/models/llava/modeling_llava.py index d5f5ee920..4e01fb719 100644 --- a/QEfficient/transformers/models/llava/modeling_llava.py +++ b/QEfficient/transformers/models/llava/modeling_llava.py @@ -5,7 +5,7 @@ # # ----------------------------------------------------------------------------- -from typing import List, Optional +from typing import List, Optional, Type import torch import torch.nn as nn @@ -30,6 +30,16 @@ def __init__(self, model): self.model = model self.model.vision_model = self.model.vision_tower + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {self.model.vision_tower.vision_model.encoder.layers[0].__class__} + def forward(self, pixel_values): # Image features image_outputs = self.model.vision_tower(pixel_values, output_hidden_states=True) @@ -54,6 +64,16 @@ def __init__(self, model): self.language_model = self.model.language_model self.lm_head = self.model.lm_head + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {self.model.language_model.layers[0].__class__} + def forward( self, input_ids, diff --git a/QEfficient/transformers/models/llava_next/modeling_llava_next.py b/QEfficient/transformers/models/llava_next/modeling_llava_next.py index 878d04a45..e09afc1c0 100755 --- a/QEfficient/transformers/models/llava_next/modeling_llava_next.py +++ b/QEfficient/transformers/models/llava_next/modeling_llava_next.py @@ -6,7 +6,7 @@ # ----------------------------------------------------------------------------- -from typing import List, Optional +from typing import List, Optional, Type import numpy as np import torch @@ -30,6 +30,16 @@ def __init__(self, model): self.model = model self.model.vision_model = self.model.vision_tower + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {self.model.vision_tower.vision_model.encoder.layers[0].__class__} + def forward(self, pixel_values, image_sizes): if pixel_values.dim() == constants.GRANITEVISION_PIXEL_VALUE_DIM: pixel_values_new = pixel_values.squeeze(0) @@ -128,6 +138,16 @@ def __init__(self, model): self.language_model = self.model.language_model self.lm_head = self.model.lm_head + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {self.model.language_model.layers[0].__class__} + def forward( self, input_ids, diff --git a/QEfficient/transformers/models/mistral/modeling_mistral.py b/QEfficient/transformers/models/mistral/modeling_mistral.py index 5edfb8f3a..6d96b87d9 100644 --- a/QEfficient/transformers/models/mistral/modeling_mistral.py +++ b/QEfficient/transformers/models/mistral/modeling_mistral.py @@ -7,7 +7,7 @@ """PyTorch Mistral model.""" -from typing import Callable, List, Optional, Tuple, Union +from typing import Callable, List, Optional, Tuple, Type, Union import torch import torch.utils.checkpoint @@ -356,6 +356,16 @@ class QEffMistralForCausalLM(MistralForCausalLM): - add new args cache idx for the kv retention """ + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {QEffMistralDecoderLayer} + def forward( self, input_ids: torch.LongTensor = None, diff --git a/QEfficient/transformers/models/mistral3/modeling_mistral3.py b/QEfficient/transformers/models/mistral3/modeling_mistral3.py index 89e19c65b..ecc65f265 100644 --- a/QEfficient/transformers/models/mistral3/modeling_mistral3.py +++ b/QEfficient/transformers/models/mistral3/modeling_mistral3.py @@ -5,7 +5,7 @@ # # ----------------------------------------------------------------------------- -from typing import List, Optional, Tuple, Union +from typing import List, Optional, Tuple, Type, Union import torch import torch.nn as nn @@ -151,6 +151,16 @@ def __init__(self, model): self.model = model self.model.vision_model = self.model.vision_tower + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {self.model.vision_tower.transformer.layers[0].__class__} + def forward(self, pixel_values): image_sizes = torch.tensor([[pixel_values.shape[2], pixel_values.shape[3]]]).repeat(pixel_values.shape[0], 1) image_features = self.model.get_image_features( @@ -168,6 +178,16 @@ def __init__(self, model): self.config = self.model.config self.language_model = self.model.language_model + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return self.model.language_model.layers[0].__class__ + def forward( self, input_ids, diff --git a/QEfficient/transformers/models/mixtral_moe/modeling_mixtral.py b/QEfficient/transformers/models/mixtral_moe/modeling_mixtral.py index 862714fea..d89ffa8a6 100644 --- a/QEfficient/transformers/models/mixtral_moe/modeling_mixtral.py +++ b/QEfficient/transformers/models/mixtral_moe/modeling_mixtral.py @@ -7,7 +7,7 @@ """PyTorch Mixtral model.""" -from typing import List, Optional, Tuple, Union +from typing import List, Optional, Tuple, Type, Union import torch import torch.nn.functional as F @@ -414,6 +414,16 @@ class QEffMixtralForCausalLM(MixtralForCausalLM): - update the hidden_states, and fix for onnx model """ + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {QeffMixtralDecoderLayer} + def forward( self, input_ids: torch.LongTensor = None, diff --git a/QEfficient/transformers/models/molmo/modeling_molmo.py b/QEfficient/transformers/models/molmo/modeling_molmo.py index 7bfa58fc0..ac4c82777 100644 --- a/QEfficient/transformers/models/molmo/modeling_molmo.py +++ b/QEfficient/transformers/models/molmo/modeling_molmo.py @@ -6,7 +6,7 @@ # ----------------------------------------------------------------------------- import math -from typing import Callable, List, Optional, Tuple, Union +from typing import Callable, List, Optional, Tuple, Type, Union import torch import torch.nn as nn @@ -568,6 +568,16 @@ def __init__(self, model): super().__init__() self.model = model + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {self.model.model.transformer.blocks[0].__class__} + def forward(self, pixel_values, image_masks, image_input_idx, valid_idx): image_features, _ = self.model.model.vision_backbone(pixel_values, image_masks) num_image, num_patch = image_features.shape[1:3] @@ -588,6 +598,16 @@ def __init__(self, model): # self.language_model = self.model.language_model self.config = self.model.config + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {self.model.model.vision_backbone.image_vit.transformer.resblocks[0].__class__} + def forward( self, input_ids, diff --git a/QEfficient/transformers/models/mpt/modeling_mpt.py b/QEfficient/transformers/models/mpt/modeling_mpt.py index c1d98c1f8..4013a27f2 100644 --- a/QEfficient/transformers/models/mpt/modeling_mpt.py +++ b/QEfficient/transformers/models/mpt/modeling_mpt.py @@ -7,7 +7,7 @@ """PyTorch MPT model.""" -from typing import Optional, Tuple, Union +from typing import Optional, Tuple, Type, Union import torch import torch.utils.checkpoint @@ -254,6 +254,16 @@ class QEffMptForCausalLM(MptForCausalLM): - add new args cache idx for the kv retention """ + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {QEffMptBlock} + def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/QEfficient/transformers/models/olmo2/modeling_olmo2.py b/QEfficient/transformers/models/olmo2/modeling_olmo2.py index 00755cae5..d5b72219c 100644 --- a/QEfficient/transformers/models/olmo2/modeling_olmo2.py +++ b/QEfficient/transformers/models/olmo2/modeling_olmo2.py @@ -5,7 +5,7 @@ # # ----------------------------------------------------------------------------- -from typing import Callable, List, Optional, Tuple, Union +from typing import Callable, List, Optional, Tuple, Type, Union import torch from torch import nn @@ -324,6 +324,16 @@ class QEffOlmo2ForCausalLM(Olmo2ForCausalLM): - add new args cache idx for the kv retention """ + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {QEffOlmo2DecoderLayer} + def forward( self, input_ids: torch.LongTensor = None, diff --git a/QEfficient/transformers/models/phi/modeling_phi.py b/QEfficient/transformers/models/phi/modeling_phi.py index 4bf2e8785..7b1382c25 100644 --- a/QEfficient/transformers/models/phi/modeling_phi.py +++ b/QEfficient/transformers/models/phi/modeling_phi.py @@ -7,7 +7,7 @@ """PyTorch Phi model.""" -from typing import Callable, List, Optional, Tuple, Union +from typing import Callable, List, Optional, Tuple, Type, Union import torch from torch import nn @@ -323,6 +323,16 @@ class QEffPhiForCausalLM(PhiForCausalLM): - update the hidden_states, and fix for onnx model """ + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {QEffPhiDecoderLayer} + def forward( self, input_ids: torch.LongTensor = None, diff --git a/QEfficient/transformers/models/phi3/modeling_phi3.py b/QEfficient/transformers/models/phi3/modeling_phi3.py index b97a0ab8d..eed2a27bd 100644 --- a/QEfficient/transformers/models/phi3/modeling_phi3.py +++ b/QEfficient/transformers/models/phi3/modeling_phi3.py @@ -7,7 +7,7 @@ """PyTorch Phi-3 model.""" -from typing import Callable, Optional, Tuple, Union +from typing import Callable, Optional, Tuple, Type, Union import torch import torch.utils.checkpoint @@ -351,6 +351,16 @@ class QEffPhi3ForCausalLM(Phi3ForCausalLM): - update the hidden_states, and fix for onnx model """ + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {QEffPhi3DecoderLayer} + def forward( self, input_ids: torch.LongTensor = None, diff --git a/QEfficient/transformers/models/pytorch_transforms.py b/QEfficient/transformers/models/pytorch_transforms.py index 9e021851b..2eaadf6c2 100644 --- a/QEfficient/transformers/models/pytorch_transforms.py +++ b/QEfficient/transformers/models/pytorch_transforms.py @@ -885,32 +885,6 @@ def apply(cls, model: nn.Module, pooling: Union[str, Callable]) -> Tuple[nn.Modu return model, transformed -def get_decoder_layer_classes_for_export(model: nn.Module) -> set: - """ - Dynamically determine which DecoderLayer classes should be exported as functions - based on the model's architecture using the existing KVCacheTransform mapping. - """ - # Define patterns that identify decoder layer classes - DECODER_LAYER_PATTERNS = ["DecoderLayer", "Block", "Layer"] - - # Get all QEff classes that are decoder layers from the existing mapping - decoder_layer_classes = set() - - for original_class, qeff_class in KVCacheTransform._module_mapping.items(): - # Check if the QEff class name contains decoder layer patterns - qeff_class_name = qeff_class.__name__ - if any(pattern in qeff_class_name for pattern in DECODER_LAYER_PATTERNS): - decoder_layer_classes.add(qeff_class) - - # Filter to only include classes that are actually used in the current model - model_decoder_classes = set() - for module in model.modules(): - if module.__class__ in decoder_layer_classes: - model_decoder_classes.add(module.__class__) - - return model_decoder_classes - - class BlockedKVAttentionTransform: _module_mapping = { QEffLlamaAttention, diff --git a/QEfficient/transformers/models/qwen2/modeling_qwen2.py b/QEfficient/transformers/models/qwen2/modeling_qwen2.py index 7c093a4b0..c9b22d261 100644 --- a/QEfficient/transformers/models/qwen2/modeling_qwen2.py +++ b/QEfficient/transformers/models/qwen2/modeling_qwen2.py @@ -7,7 +7,7 @@ """PyTorch Qwen2 model.""" -from typing import List, Optional, Tuple, Union +from typing import List, Optional, Tuple, Type, Union import torch import torch.utils.checkpoint @@ -350,6 +350,16 @@ class QEffQwen2ForCausalLM(Qwen2ForCausalLM): - update the hidden_states, and fix for onnx model """ + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {QEffQwen2DecoderLayer} + def forward( self, input_ids: torch.LongTensor = None, diff --git a/QEfficient/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py b/QEfficient/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py index 63e046600..9c284dd1b 100644 --- a/QEfficient/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py +++ b/QEfficient/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py @@ -7,7 +7,7 @@ import math import os -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union import torch import torch.nn as nn @@ -73,14 +73,10 @@ def qeff_apply_rotary_pos_emb(q, k, cos, sin, position_ids, mrope_section, unsqu Returns: `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. """ - - mrope_section = mrope_section * 2 cos = cos[position_ids] sin = sin[position_ids] - - cos = torch.cat([m[i % 3] for i, m in enumerate(cos.split(mrope_section, dim=-1))], dim=-1).unsqueeze(unsqueeze_dim) - sin = torch.cat([m[i % 3] for i, m in enumerate(sin.split(mrope_section, dim=-1))], dim=-1).unsqueeze(unsqueeze_dim) - + cos = torch.cat([cos[0, ..., 0:32], cos[0, ..., 32:80], cos[0, ..., 80:128]], dim=-1).unsqueeze(0) + sin = torch.cat([sin[0, ..., 0:32], sin[0, ..., 32:80], sin[0, ..., 80:128]], dim=-1).unsqueeze(0) q_embed = (q * cos) + (rotate_half(q) * sin) k_embed = (k * cos) + (rotate_half(k) * sin) @@ -872,6 +868,16 @@ def __init__(self, model): self.model = model self.model.vision_model = self.model.visual + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {self.model.visual.blocks[0].__class__} + def forward(self, pixel_values, image_grid_thw): image_embeds = self.model.visual(pixel_values, grid_thw=image_grid_thw) bs = image_grid_thw.shape[0] @@ -887,6 +893,16 @@ def __init__(self, model): self.model = model self.language_model = self.model.model.language_model + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {QEffQwen2_5_VLDecoderLayer} + def forward( self, input_ids, diff --git a/QEfficient/transformers/models/qwen3/modeling_qwen3.py b/QEfficient/transformers/models/qwen3/modeling_qwen3.py index 540bad4c7..919f49166 100644 --- a/QEfficient/transformers/models/qwen3/modeling_qwen3.py +++ b/QEfficient/transformers/models/qwen3/modeling_qwen3.py @@ -7,7 +7,7 @@ """PyTorch Qwen3 model.""" -from typing import List, Optional, Tuple, Union +from typing import List, Optional, Tuple, Type, Union import torch import torch.utils.checkpoint @@ -351,6 +351,16 @@ class QEffQwen3ForCausalLM(Qwen3ForCausalLM): - update the hidden_states, and fix for onnx model """ + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {QEffQwen3DecoderLayer} + def forward( self, input_ids: torch.LongTensor = None, diff --git a/QEfficient/transformers/models/qwen3_moe/modeling_qwen3_moe.py b/QEfficient/transformers/models/qwen3_moe/modeling_qwen3_moe.py index cbd80d8ca..8140056c5 100644 --- a/QEfficient/transformers/models/qwen3_moe/modeling_qwen3_moe.py +++ b/QEfficient/transformers/models/qwen3_moe/modeling_qwen3_moe.py @@ -5,7 +5,7 @@ # # ----------------------------------------------------------------------------- -from typing import List, Optional, Tuple +from typing import List, Optional, Tuple, Type import torch import torch.nn.functional as F @@ -371,6 +371,16 @@ def forward( class QEffQwen3MoeForCausalLM(Qwen3MoeForCausalLM): + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {QEffQwen3MoeDecoderLayer} + def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/QEfficient/transformers/models/starcoder2/modeling_starcoder2.py b/QEfficient/transformers/models/starcoder2/modeling_starcoder2.py index c86e7478b..310c31254 100644 --- a/QEfficient/transformers/models/starcoder2/modeling_starcoder2.py +++ b/QEfficient/transformers/models/starcoder2/modeling_starcoder2.py @@ -7,7 +7,7 @@ """PyTorch Starcoder2 model.""" -from typing import List, Optional, Tuple, Union +from typing import List, Optional, Tuple, Type, Union import torch from torch import nn @@ -275,6 +275,16 @@ class QEffStarcoder2ForCausalLM(Starcoder2ForCausalLM): - update the hidden_states, and fix for onnx model """ + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {QEFFStarcoder2DecoderLayer} + def forward( self, input_ids: torch.LongTensor = None, diff --git a/QEfficient/transformers/models/whisper/modeling_whisper.py b/QEfficient/transformers/models/whisper/modeling_whisper.py index a03ffecf7..2257ec06b 100644 --- a/QEfficient/transformers/models/whisper/modeling_whisper.py +++ b/QEfficient/transformers/models/whisper/modeling_whisper.py @@ -5,7 +5,7 @@ # # ---------------------------------------------------------------------------- -from typing import Optional, Tuple, Union +from typing import Optional, Tuple, Type, Union import torch from torch import nn @@ -718,6 +718,16 @@ class QEffWhisperForConditionalGeneration(WhisperForConditionalGeneration): - changed forward inputs decoder_input_ids and decoder_position_ids to input_ids and position_ids """ + def get_repeated_layer_class(self) -> Type[nn.Module]: + """ + Return the set of class used as the repeated layer across the model for subfunction extraction. + + Notes: + This method should return the *class object* (not an instance). + Downstream code can use this to find/build subfunctions for repeated blocks. + """ + return {self.model.encoder.layers[0].__class__, QEffWhisperDecoderLayer} + def forward( self, input_features: Optional[torch.FloatTensor] = None, diff --git a/QEfficient/utils/export_utils.py b/QEfficient/utils/export_utils.py index 638f55921..c47cc7f70 100644 --- a/QEfficient/utils/export_utils.py +++ b/QEfficient/utils/export_utils.py @@ -14,7 +14,6 @@ from QEfficient.base.onnx_transforms import CustomOpTransform, RenameFunctionOutputsTransform from QEfficient.transformers.cache_utils import InvalidIndexProvider -from QEfficient.transformers.models.pytorch_transforms import get_decoder_layer_classes_for_export from QEfficient.utils.cache import QEFF_HOME from QEfficient.utils.hash_utils import create_export_hash from QEfficient.utils.logging_utils import logger @@ -164,18 +163,27 @@ def _setup_onnx_subfunctions(qeff_model, args, kwargs): # Transform output names for subfunction compatibility if "output_names" in kwargs: kwargs["output_names"] = [ - re.sub("_RetainedState", "_InternalRetainedState", name) for name in kwargs["output_names"] + re.sub("_RetainedState", "_InternalRetainedState", name) + if name.endswith("_RetainedState") and ("key" in name or "value" in name) + else name + for name in kwargs["output_names"] ] else: args = list(args) - args[1] = [re.sub("_RetainedState", "_InternalRetainedState", name) for name in args[1]] + args[1] = [ + re.sub("_RetainedState", "_InternalRetainedState", name) + if name.endswith("_RetainedState") and ("key" in name or "value" in name) + else name + for name in args[1] + ] args = tuple(args) + # Add subfunction-specific ONNX transforms qeff_model._onnx_transforms.append(RenameFunctionOutputsTransform) qeff_model._onnx_transforms.append(CustomOpTransform) # TODO: Handle this in the modelling class QEFFTransformersBase,remove from here. Refer diffusers implementation - kwargs["export_modules_as_functions"] = get_decoder_layer_classes_for_export(qeff_model.model) + kwargs["export_modules_as_functions"] = qeff_model.model.get_repeated_layer_class() return args, kwargs diff --git a/QEfficient/utils/torch_patches.py b/QEfficient/utils/torch_patches.py index 0b9b37afa..241b32fbf 100644 --- a/QEfficient/utils/torch_patches.py +++ b/QEfficient/utils/torch_patches.py @@ -7,6 +7,8 @@ """Monkey patches for torch.onnx.utils to fix ONNX export issues.""" +import warnings + import torch import torch.onnx.utils as onnx_utils from torch import _C @@ -37,9 +39,13 @@ def _track_module_attributes_forward_hook(module, input, output): if hasattr(module, attr_name): onnx_attrs = getattr(module, attr_name) delattr(module, attr_name) + # FIX: use empty dict to avoid type mismatch - onnx_attrs = {} - _C._jit_pass_onnx_track_scope_attributes(graph, onnx_attrs) + # onnx_attrs = {} + try: + _C._jit_pass_onnx_track_scope_attributes(graph, onnx_attrs) + except Exception as e: + warnings.warn(f"Failed to track ONNX scope attributes: {e}. Skipping this step.") for m in model.modules(): m.register_forward_hook(_track_module_attributes_forward_hook)