|
| 1 | +"""Boogu-Image text encoder: full Qwen3-VL-8B, last hidden state (4096-dim). |
| 2 | +
|
| 3 | +Boogu uses the final hidden state of Qwen3-VL as the per-token instruction feature |
| 4 | +(num_instruction_feature_layers=1, reduce_type=mean -> just the last layer). |
| 5 | +The model itself is the standard Qwen3-VL TE, only the chat template differs |
| 6 | +(a fixed system prompt and no <think> block). |
| 7 | +""" |
| 8 | + |
| 9 | +import comfy.text_encoders.qwen3vl |
| 10 | +from comfy import sd1_clip |
| 11 | + |
| 12 | + |
| 13 | +# System prompts from the reference pipeline (pipeline_boogu.py). |
| 14 | +# T2I (non-empty instruction, no image) uses the helpful-assistant prompt |
| 15 | +# everything else (the CFG negative / "drop" condition, and any image case) uses the TI2I "describe" prompt. |
| 16 | +BOOGU_T2I_SYSTEM = "You are a helpful assistant that generates high-quality images based on user instructions. The instructions are as follows." |
| 17 | +BOOGU_DROP_SYSTEM = "Describe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate." |
| 18 | + |
| 19 | + |
| 20 | +class BooguTokenizer(comfy.text_encoders.qwen3vl.Qwen3VLTokenizer): |
| 21 | + def __init__(self, embedding_directory=None, tokenizer_data={}): |
| 22 | + super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, model_type="qwen3vl_8b") |
| 23 | + # apply_chat_template without add_generation_prompt |
| 24 | + self.llama_template = "<|im_start|>system\n" + BOOGU_T2I_SYSTEM + "<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n" |
| 25 | + self.llama_template_images = "<|im_start|>system\n" + BOOGU_DROP_SYSTEM + "<|im_end|>\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>{}<|im_end|>\n" |
| 26 | + # Reference SYSTEM_PROMPT_DROP: used for the empty negative/uncond instruction. |
| 27 | + self.llama_template_drop = "<|im_start|>system\n" + BOOGU_DROP_SYSTEM + "<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n" |
| 28 | + |
| 29 | + def tokenize_with_weights(self, text, return_word_ids=False, llama_template=None, images=[], prevent_empty_text=False, thinking=True, **kwargs): |
| 30 | + if llama_template is None and len(images) == 0 and text.strip() == "": |
| 31 | + llama_template = self.llama_template_drop |
| 32 | + # Boogu conditions on the no-think template; thinking=True drops the empty <think> block qwen3vl adds by default. |
| 33 | + return super().tokenize_with_weights(text, return_word_ids=return_word_ids, llama_template=llama_template, images=images, prevent_empty_text=prevent_empty_text, thinking=thinking, **kwargs) |
| 34 | + |
| 35 | + |
| 36 | +class BooguQwen3VLClipModel(comfy.text_encoders.qwen3vl.Qwen3VLClipModel): |
| 37 | + def __init__(self, device="cpu", dtype=None, attention_mask=True, model_options={}, model_type="qwen3vl_8b"): |
| 38 | + super().__init__(device=device, dtype=dtype, attention_mask=attention_mask, model_options=model_options, model_type=model_type) |
| 39 | + # apply the final RMSNorm to the tapped last layer |
| 40 | + self.layer_norm_hidden_state = True |
| 41 | + |
| 42 | + |
| 43 | +class BooguTEModel(sd1_clip.SD1ClipModel): |
| 44 | + def __init__(self, device="cpu", dtype=None, model_options={}): |
| 45 | + clip_model = lambda **kw: BooguQwen3VLClipModel(**kw, model_type="qwen3vl_8b") |
| 46 | + super().__init__(device=device, dtype=dtype, name="qwen3vl_8b", clip_model=clip_model, model_options=model_options) |
| 47 | + |
| 48 | + |
| 49 | +def te(dtype_llama=None, llama_quantization_metadata=None): |
| 50 | + class BooguTEModel_(BooguTEModel): |
| 51 | + def __init__(self, device="cpu", dtype=None, model_options={}): |
| 52 | + if dtype_llama is not None: |
| 53 | + dtype = dtype_llama |
| 54 | + if llama_quantization_metadata is not None: |
| 55 | + model_options = model_options.copy() |
| 56 | + model_options["quantization_metadata"] = llama_quantization_metadata |
| 57 | + super().__init__(device=device, dtype=dtype, model_options=model_options) |
| 58 | + return BooguTEModel_ |
0 commit comments