added config (torch) to app

InstructionTextGenerationPipeline.py

@@ -27,7 +27,13 @@ PROMPT_FOR_GENERATION_FORMAT = """{intro}
     instruction="{instruction}",
     response_key=RESPONSE_KEY,
 )
-
+#
+#generate = InstructionTextGenerationPipeline(
+#    "mosaicml/mpt-7b-instruct",
+#    torch_dtype=torch.bfloat16,
+#    trust_remote_code=True,
+#    config=config,
+#)
 
 class InstructionTextGenerationPipeline:
     def __init__(
@@ -35,10 +41,12 @@ class InstructionTextGenerationPipeline:
         model_name,
         torch_dtype=torch.bfloat16,
         trust_remote_code=True,
+        config=None,
         use_auth_token=None,
     ) -> None:
         self.model = AutoModelForCausalLM.from_pretrained(
             model_name,
+            config=config,
             torch_dtype=torch_dtype,
             trust_remote_code=trust_remote_code,
             use_auth_token=use_auth_token,

app.py

@@ -62,10 +62,17 @@ def complete(state="complete"):
 complete(state='imports done')
 
 complete(state="start generate")
+
+name = 'mosaicml/mpt-7b-instruct'
+config = transformers.AutoConfig.from_pretrained(name, trust_remote_code=True)
+config.attn_config['attn_impl'] = 'torch'
+config.init_device = 'cuda:0' # For fast initialization directly on GPU!
+
 generate = InstructionTextGenerationPipeline(
-    "mosaicml/mpt-7b-instruct",
+    name,
     torch_dtype=torch.bfloat16,
     trust_remote_code=True,
+    config=config,
 )
 stop_token_ids = generate.tokenizer.convert_tokens_to_ids(["<|endoftext|>"])
 complete(state="Model generated")
@@ -134,6 +141,9 @@ def tester(uPrompt, max_new_tokens, temperature, top_k, top_p):
     results = f"{salutation} max_new_tokens{max_new_tokens}; temperature{temperature}; top_k{top_k}; top_p{top_p};  "
     
     return response
+import torch
+import transformers
+
 demo = gr.Interface(
     fn=tester,
     inputs=[gr.Textbox(label="Prompt",info="Prompt",lines=3,value="Provide Prompt"),  
@@ -144,7 +154,7 @@ demo = gr.Interface(
            
            ],
     outputs=["text"],
-    title="Mosais MPT-7B",
+    title="Mosaic MPT-7B",
 )
 demo.launch(share=True,
     server_name="0.0.0.0",

nohup.out

@@ -1,51 +1,17 @@
-sys.path : ['/home/ubuntu/Gradio/MPT7BTest', '/usr/lib/python310.zip', '/usr/lib/python3.10', '/usr/lib/python3.10/lib-dynload', '/home/ubuntu/.local/lib/python3.10/site-packages', '/usr/local/lib/python3.10/dist-packages', '/usr/lib/python3/dist-packages', '/home/ec2-user/workspace/Notebooks/lib']
+sys.path : ['/home/ec2-user/Gradio/MPT7BTest', '/usr/lib64/python37.zip', '/usr/lib64/python3.7', '/usr/lib64/python3.7/lib-dynload', '/home/ec2-user/.local/lib/python3.7/site-packages', '/usr/lib64/python3.7/site-packages', '/usr/lib/python3.7/site-packages', '/home/ec2-user/workspace/Notebooks/lib']
 
 Cell imports done
 
 Cell start generate
 You are using config.init_device='cpu', but you can also use config.init_device="meta" with Composer + FSDP for fast initialization.
-
+
+/home/ec2-user/Gradio/MPT7BTest/InstructionTextGenerationPipeline.py:54: UserWarning: pad_token_id is not set for the tokenizer. Using eos_token_id as pad_token_id.
+  "pad_token_id is not set for the tokenizer. Using eos_token_id as pad_token_id."
 
-Cell imports done
-
-Cell start generate
-You are using config.init_device='cpu', but you can also use config.init_device="meta" with Composer + FSDP for fast initialization.
-
+Cell Model generated
+Running on local URL:  http://0.0.0.0:7860
+Running on public URL: https://02eb0eb5bb97dfa857.gradio.live
 
-Cell imports done
-
-Cell start generate
-You are using config.init_device='cpu', but you can also use config.init_device="meta" with Composer + FSDP for fast initialization.
-Traceback (most recent call last):
-  File "/home/ubuntu/Gradio/MPT7BTest/app.py", line 65, in <module>
-    generate = InstructionTextGenerationPipeline(
-  File "/home/ubuntu/Gradio/MPT7BTest/InstructionTextGenerationPipeline.py", line 40, in __init__
-    self.model = AutoModelForCausalLM.from_pretrained(
-  File "/home/ubuntu/.local/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py", line 479, in from_pretrained
-    return model_class.from_pretrained(
-  File "/home/ubuntu/.local/lib/python3.10/site-packages/transformers/modeling_utils.py", line 2675, in from_pretrained
-    model = cls(config, *model_args, **model_kwargs)
-  File "/home/ubuntu/.cache/huggingface/modules/transformers_modules/mosaicml/mpt-7b-instruct/e7119f37956c1a3865da33e25ef5ce9159ff2c16/modeling_mpt.py", line 222, in __init__
-    self.transformer = MPTModel(config)
-  File "/home/ubuntu/.cache/huggingface/modules/transformers_modules/mosaicml/mpt-7b-instruct/e7119f37956c1a3865da33e25ef5ce9159ff2c16/modeling_mpt.py", line 55, in __init__
-    self.apply(self.param_init_fn)
-  File "/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 884, in apply
-    module.apply(fn)
-  File "/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 884, in apply
-    module.apply(fn)
-  File "/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 884, in apply
-    module.apply(fn)
-  [Previous line repeated 1 more time]
-  File "/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 885, in apply
-    fn(self)
-  File "/home/ubuntu/.cache/huggingface/modules/transformers_modules/mosaicml/mpt-7b-instruct/e7119f37956c1a3865da33e25ef5ce9159ff2c16/modeling_mpt.py", line 208, in param_init_fn
-    MODEL_INIT_REGISTRY[init_fn_name](module=module, n_layers=self.config.n_layers, d_model=self.config.d_model, **self.config.init_config)
-  File "/home/ubuntu/.cache/huggingface/modules/transformers_modules/mosaicml/mpt-7b-instruct/e7119f37956c1a3865da33e25ef5ce9159ff2c16/param_init_fns.py", line 167, in kaiming_normal_param_init_fn_
-    generic_param_init_fn_(module=module, init_fn_=kaiming_normal_, d_model=d_model, n_layers=n_layers, init_div_is_residual=init_div_is_residual, emb_init_std=emb_init_std, emb_init_uniform_lim=emb_init_uniform_lim, verbose=verbose)
-  File "/home/ubuntu/.cache/huggingface/modules/transformers_modules/mosaicml/mpt-7b-instruct/e7119f37956c1a3865da33e25ef5ce9159ff2c16/param_init_fns.py", line 49, in generic_param_init_fn_
-    fused_init_helper_(module, init_fn_)
-  File "/home/ubuntu/.cache/huggingface/modules/transformers_modules/mosaicml/mpt-7b-instruct/e7119f37956c1a3865da33e25ef5ce9159ff2c16/param_init_fns.py", line 26, in fused_init_helper_
-    init_fn_(module.weight[slice_indices])
-  File "/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/init.py", line 451, in kaiming_normal_
-    return tensor.normal_(0, std)
-KeyboardInterrupt
+This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces)
+Keyboard interruption in main thread... closing server.
+Killing tunnel 0.0.0.0:7860 <> https://02eb0eb5bb97dfa857.gradio.live