support external custom models

Author: qiuxuan.lzw

python/sglang/srt/configs/model_config.py

@@ -957,6 +957,9 @@ def is_generation_model(model_architectures: List[str], is_embedding: bool = Fal
     "DeepseekOCRForCausalLM",
 ]
 
+if envs.SGLANG_EXTERNAL_MM_MODEL_ARCH.value:
+    multimodal_model_archs.append(envs.SGLANG_EXTERNAL_MM_MODEL_ARCH.value)
+
 
 def is_multimodal_model(model_architectures: List[str]):
     if any(

python/sglang/srt/environ.py

@@ -295,6 +295,11 @@ class Envs:
     # Health Check
     SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION = EnvBool(True)
 
+    # External models
+    SGLANG_EXTERNAL_MODEL_PACKAGE = EnvStr("")
+    SGLANG_EXTERNAL_MM_MODEL_ARCH = EnvStr("")
+    SGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE = EnvStr("")
+
     # fmt: on
 
 

python/sglang/srt/managers/multimodal_processor.py

@@ -12,7 +12,7 @@
 PROCESSOR_MAPPING = {}
 
 
-def import_processors(package_name: str):
+def import_processors(package_name: str, overwrite: bool = False):
     package = importlib.import_module(package_name)
     for _, name, ispkg in pkgutil.iter_modules(package.__path__, package_name + "."):
         if not ispkg:
@@ -32,6 +32,11 @@ def import_processors(package_name: str):
             ):
                 assert hasattr(cls, "models")
                 for arch in getattr(cls, "models"):
+                    if overwrite:
+                        for model_cls, processor_cls in PROCESSOR_MAPPING.items():
+                            if model_cls.__name__ == arch.__name__:
+                                del PROCESSOR_MAPPING[model_cls]
+                                break
                     PROCESSOR_MAPPING[arch] = cls
 
 

python/sglang/srt/managers/tokenizer_manager.py

@@ -41,6 +41,7 @@
 
 from sglang.srt.configs.model_config import ModelConfig
 from sglang.srt.disaggregation.utils import DisaggregationMode
+from sglang.srt.environ import envs
 from sglang.srt.lora.lora_registry import LoRARegistry
 from sglang.srt.managers.async_dynamic_batch_tokenizer import AsyncDynamicbatchTokenizer
 from sglang.srt.managers.async_mm_data_processor import AsyncMMDataProcessor
@@ -210,6 +211,10 @@ def __init__(
         # Initialize tokenizer and processor
         if self.model_config.is_multimodal:
             import_processors("sglang.srt.multimodal.processors")
+            if envs.SGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE.value:
+                import_processors(
+                    envs.SGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE.value, overwrite=True
+                )
             try:
                 _processor = get_processor(
                     server_args.tokenizer_path,

python/sglang/srt/models/registry.py

@@ -123,3 +123,6 @@ def import_model_classes(package_name: str):
 
 ModelRegistry = _ModelRegistry()
 ModelRegistry.register("sglang.srt.models")
+
+if envs.SGLANG_EXTERNAL_MODEL_PACKAGE.value:
+    ModelRegistry.register(envs.SGLANG_EXTERNAL_MODEL_PACKAGE.value, overwrite=True)

test/srt/external_models/custom_qwen2_vl.py

@@ -0,0 +1,21 @@
+from sglang.srt.models.qwen2_vl import (
+    Qwen2VLForConditionalGeneration as OriginalQwen2VLForConditionalGeneration,
+)
+from sglang.srt.multimodal.processors.qwen_vl import QwenVLImageProcessor
+
+
+class Qwen2VLForConditionalGeneration(OriginalQwen2VLForConditionalGeneration):
+    def __init__(self, config, quant_config, prefix: str = "") -> None:
+        super().__init__(config, quant_config, prefix)
+        print("init custom model:", self.__class__.__name__)
+
+
+class CustomProcessor(QwenVLImageProcessor):
+    models = [Qwen2VLForConditionalGeneration]
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+        print("init custom processor:", self.__class__.__name__)
+
+
+EntryClass = Qwen2VLForConditionalGeneration

test/srt/test_external_models.py

@@ -0,0 +1,28 @@
+import os
+import unittest
+
+import sglang as sgl
+from sglang.test.test_utils import CustomTestCase
+
+
+class TestExternalModels(CustomTestCase):
+    def test_external_model(self):
+        os.environ["SGLANG_EXTERNAL_MODEL_PACKAGE"] = "external_models"
+        os.environ["SGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE"] = "external_models"
+        prompt = "Today is a sunny day and I like"
+        model_path = "Qwen/Qwen2-VL-2B-Instruct"
+
+        engine = sgl.Engine(
+            model_path=model_path,
+            cuda_graph_max_bs=1,
+            max_total_tokens=64,
+            enable_multimodal=True,
+        )
+        out = engine.generate(prompt)["text"]
+        engine.shutdown()
+
+        self.assertGreater(len(out), 0)
+
+
+if __name__ == "__main__":
+    unittest.main()