Update modeling_qwen.py

modeling_qwen.py

@@ -564,7 +564,13 @@ class QWenModel(QWenPreTrainedModel):
 
             images = self.visual.encode(images)
             assert images.shape[0] == len(images)
+            fake_images = None
+        elif self.training:
+            fake_images=torch.zeros(1,3,224,224).to(
+                dtype=self.visual.conv1.weight.dtype, device=self.visual.conv1.weight.device)
+            images = self.visual(fake_images)
         else:
+            fake_images = None
             images = None
 
         output_attentions = (
@@ -623,11 +629,6 @@ class QWenModel(QWenPreTrainedModel):
 
         if inputs_embeds is None:
             inputs_embeds = self.wte(input_ids)
-            if self.training and images == None: # Compatible with plain text data training
-                fake_images=torch.zeros(1,3,224,224).to(
-                        dtype=self.visual.conv1.weight.dtype, device=self.visual.conv1.weight.device)
-                image_embeds = self.visual(fake_images)
-                inputs_embeds = inputs_embeds + image_embeds.mean()*0
 
         if batch_size <= 0:
             raise ValueError("batch_size has to be defined and > 0")
@@ -657,7 +658,9 @@ class QWenModel(QWenPreTrainedModel):
             rotary_pos_emb[idx] = rotary_pos_emb[idx].to(hidden_states.device)
 
         hidden_states = self.drop(hidden_states).clone()
-        if images is not None:
+        if fake_images is not None:
+            hidden_states = hidden_states + images.mean()*0
+        elif images is not None:
             for idx, (i, a, b) in enumerate(img_pos):
                 hidden_states[i][a + 1 : b] = images[idx]
         output_shape = input_shape + (hidden_states.size(-1),)