updated version of ui

app.py

@@ -59,26 +59,28 @@ def check_dataset(dataset_id, dataset_config="default", dataset_split="test"):
         return dataset_id, None, None
     return dataset_id, dataset_config, dataset_split
 
-def try_validate(model_id, dataset_id, dataset_config, dataset_split, column_mapping):
+def try_validate(model_id, dataset_id, dataset_config, dataset_split, column_mapping='{}'):
     # Validate model
     m_id, ppl = check_model(model_id=model_id)
     if m_id is None:
         gr.Warning(f'Model "{model_id}" is not accessible. Please set your HF_TOKEN if it is a private model.')
         return (
-            dataset_config, dataset_split,
             gr.update(interactive=False),   # Submit button
+            gr.update(visible=True),       # Loading row
+            gr.update(visible=False),        # Preview row
+            gr.update(visible=False),       # Model prediction input
             gr.update(visible=False),       # Model prediction preview
             gr.update(visible=False),       # Label mapping preview
-            gr.update(visible=True),        # Column mapping
         )
     if isinstance(ppl, Exception):
         gr.Warning(f'Failed to load "{model_id} model": {ppl}')
         return (
-            dataset_config, dataset_split,
             gr.update(interactive=False),   # Submit button
+            gr.update(visible=True),       # Loading row
+            gr.update(visible=False),        # Preview row
+            gr.update(visible=False),       # Model prediction input
             gr.update(visible=False),       # Model prediction preview
             gr.update(visible=False),       # Label mapping preview
-            gr.update(visible=True),        # Column mapping
         )
 
     # Validate dataset
@@ -98,11 +100,13 @@ def try_validate(model_id, dataset_id, dataset_config, dataset_split, column_map
 
     if not dataset_ok:
         return (
-            config, split,
             gr.update(interactive=False),   # Submit button
+            gr.update(visible=True),       # Loading row
+            gr.update(visible=False),        # Preview row
+            gr.update(visible=False),       # Model prediction input
             gr.update(visible=False),       # Model prediction preview
             gr.update(visible=False),       # Label mapping preview
-            gr.update(visible=True),        # Column mapping
+            # gr.update(visible=True),        # Column mapping
         )
 
     # TODO: Validate column mapping by running once
@@ -110,11 +114,12 @@ def try_validate(model_id, dataset_id, dataset_config, dataset_split, column_map
     id2label_df = None
     if isinstance(ppl, TextClassificationPipeline):
         try:
+            print('validating phase, ', column_mapping)
             column_mapping = json.loads(column_mapping)
         except Exception:
             column_mapping = {}
 
-        column_mapping, prediction_result, id2label_df = \
+        column_mapping, prediction_input, prediction_result, id2label_df = \
             text_classification_fix_column_mapping(column_mapping, ppl, d_id, config, split)
 
         column_mapping = json.dumps(column_mapping, indent=2)
@@ -124,31 +129,35 @@ def try_validate(model_id, dataset_id, dataset_config, dataset_split, column_map
     if prediction_result is None:
         gr.Warning('The model failed to predict with the first row in the dataset. Please provide column mappings in "Advance" settings.')
         return (
-            config, split,
             gr.update(interactive=False),   # Submit button
+            gr.update(visible=True),       # Loading row
+            gr.update(visible=False),        # Preview row
+            gr.update(visible=False),       # Model prediction input
             gr.update(visible=False),   # Model prediction preview
             gr.update(visible=False),   # Label mapping preview
-            gr.update(value=column_mapping, visible=True, interactive=True),    # Column mapping
+            # gr.update(value=column_mapping, visible=True, interactive=True),    # Column mapping
         )
     elif id2label_df is None:
         gr.Warning('The prediction result does not conform the labels in the dataset. Please provide label mappings in "Advance" settings.')
         return (
-            config, split,
             gr.update(interactive=False),   # Submit button
+            gr.update(visible=False),       # Loading row
+            gr.update(visible=True),        # Preview row
+            gr.update(value=f'**Sample Input**: {prediction_input}', visible=True),       # Model prediction input
             gr.update(value=prediction_result, visible=True),   # Model prediction preview
             gr.update(visible=False),   # Label mapping preview
-            gr.update(value=column_mapping, visible=True, interactive=True),    # Column mapping
+            # gr.update(value=column_mapping, visible=True, interactive=True),    # Column mapping
         )
 
     gr.Info("Model and dataset validations passed. Your can submit the evaluation task.")
 
     return (
+        gr.update(interactive=True),    # Submit button
         gr.update(visible=False),       # Loading row
         gr.update(visible=True),        # Preview row
-        gr.update(interactive=True),    # Submit button
+        gr.update(value=f'**Sample Input**: {prediction_input}', visible=True),       # Model prediction input
         gr.update(value=prediction_result, visible=True),   # Model prediction preview
-        gr.update(value=id2label_df, visible=True), # Label mapping preview
-        gr.update(value=column_mapping, visible=True, interactive=True),    # Column mapping
+        gr.update(value=id2label_df, visible=True, interactive=True), # Label mapping preview
     )
 
 
@@ -200,10 +209,7 @@ def try_submit(m_id, d_id, config, split, column_mappings, local):
 
 with gr.Blocks(theme=theme) as iface:
     with gr.Tab("Text Classification"):
-        global_ds_id = gr.State('ds')
-
         def check_dataset_and_get_config(dataset_id):
-            global_ds_id.value = dataset_id
             try:
                 configs = datasets.get_dataset_config_names(dataset_id)
                 print(configs)
@@ -212,10 +218,9 @@ with gr.Blocks(theme=theme) as iface:
                 # Dataset may not exist
                 pass
 
-        def check_dataset_and_get_split(choice):
-            print('choice: ',choice, global_ds_id.value)
+        def check_dataset_and_get_split(dataset_config, dataset_id):
             try:
-                splits = list(datasets.load_dataset(global_ds_id.value, choice).keys())
+                splits = list(datasets.load_dataset(dataset_id, dataset_config).keys())
                 print('splits: ',splits)
                 return gr.Dropdown(splits, value=splits[0], visible=True)
             except Exception as e:
@@ -223,12 +228,20 @@ with gr.Blocks(theme=theme) as iface:
                 print(e)
                 pass
         
-        def gate_validate_btn(model_id, dataset_id, dataset_config, dataset_split):
+        def gate_validate_btn(model_id, dataset_id, dataset_config, dataset_split, id2label_mapping_dataframe=None):
             print('model_id: ',model_id)
+            column_mapping = '{}'
+            if id2label_mapping_dataframe is not None:
+                column_mapping = id2label_mapping_dataframe.to_json(orient="split")
+                print(column_mapping)   
             if model_id and dataset_id and dataset_config and dataset_split:
-                return gr.update(interactive=True)
+                return try_validate(model_id, dataset_id, dataset_config, dataset_split, column_mapping)
             else:
-                return gr.update(interactive=False)
+                return (gr.update(interactive=False), 
+                        gr.update(visible=True),
+                        gr.update(visible=False),
+                        gr.update(visible=False),
+                        gr.update(visible=False))
 
         with gr.Row():
             model_id_input = gr.Textbox(
@@ -245,22 +258,10 @@ with gr.Blocks(theme=theme) as iface:
             dataset_split_input = gr.Dropdown(['default'], value=['default'], label='Dataset Split', visible=False)
 
             dataset_id_input.change(check_dataset_and_get_config, dataset_id_input, dataset_config_input)
-            dataset_config_input.change(check_dataset_and_get_split, dataset_config_input, dataset_split_input)
-
-        with gr.Row():
-            validate_btn = gr.Button("Validate Model and Dataset", variant="primary", interactive=False)
-            model_id_input.change(gate_validate_btn, 
-                                  inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input], 
-                                  outputs=[validate_btn])
-            dataset_id_input.change(gate_validate_btn, 
-                                  inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input], 
-                                  outputs=[validate_btn])
-            dataset_config_input.change(gate_validate_btn,
-                                    inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input],
-                                    outputs=[validate_btn])
-            dataset_split_input.change(gate_validate_btn,
-                                    inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input],
-                                    outputs=[validate_btn])
+            dataset_config_input.change(
+                check_dataset_and_get_split, 
+                inputs=[dataset_config_input, dataset_id_input], 
+                outputs=[dataset_split_input])
         
         with gr.Row(visible=True) as loading_row:
             gr.Markdown('''
@@ -270,51 +271,45 @@ with gr.Blocks(theme=theme) as iface:
                         ''')
 
         with gr.Row(visible=False) as preview_row:
-            with gr.Column():
-                id2label_mapping_dataframe = gr.DataFrame(label="Preview of label mapping")
-                
-                gr.Markdown('''
-                            <span style="background-color:#5fc269; color:white">Does this look right? If not, Check and update your feature mapping -></span>
-                            ''')
-
-                example_labels = gr.Label(label='Model Prediction Sample')
-
-
-            with gr.Accordion("Advance", open=False):
-                run_local = gr.Checkbox(value=True, label="Run in this Space")
-                column_mapping_input = gr.Textbox(
-                    value="",
-                    lines=6,
-                    label="Column mapping",
-                    placeholder="Description of mapping of columns in model to dataset, in json format, e.g.:\n"
-                                '{\n'
-                                '   "text": "context",\n'
-                                '   "label": {0: "Positive", 1: "Negative"}\n'
-                                '}',
-                )
+            gr.Markdown('''
+                <h1 style="text-align: center;">
+                Confirm Label Details
+                </h1>
+                Base on your model and dataset, we inferred this label mapping. **If the mapping is incorrect, please modify it in the table below.**
+                ''')
+        
+        with gr.Row():
+            id2label_mapping_dataframe = gr.DataFrame(label="Preview of label mapping", interactive=True, visible=False)
+
+        with gr.Row():
+            example_input = gr.Markdown('Sample Input: ', visible=False)
+        
+        with gr.Row():
+            example_labels = gr.Label(label='Model Prediction Sample', visible=False)
 
+            
         run_btn = gr.Button(
             "Get Evaluation Result",
             variant="primary",
             interactive=False,
+            size="lg",
         )
-        validate_btn.click(
-            try_validate,
-            inputs=[
-                model_id_input,
-                dataset_id_input,
-                dataset_config_input,
-                dataset_split_input,
-            ],
-            outputs=[
-                loading_row,
-                preview_row,
-                run_btn,
-                example_labels,
-                id2label_mapping_dataframe,
-                column_mapping_input,
-            ],
-        )
+
+        model_id_input.change(gate_validate_btn, 
+                                inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input], 
+                                outputs=[run_btn, loading_row, preview_row, example_input, example_labels, id2label_mapping_dataframe])
+        dataset_id_input.change(gate_validate_btn, 
+                                inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input], 
+                                outputs=[run_btn, loading_row, preview_row, example_input,  example_labels, id2label_mapping_dataframe])
+        dataset_config_input.change(gate_validate_btn,
+                                inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input],
+                                outputs=[run_btn, loading_row, preview_row, example_input, example_labels, id2label_mapping_dataframe])
+        dataset_split_input.change(gate_validate_btn,
+                                inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input],
+                                outputs=[run_btn, loading_row, preview_row, example_input, example_labels, id2label_mapping_dataframe])
+        id2label_mapping_dataframe.input(gate_validate_btn,
+                                inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input, id2label_mapping_dataframe],
+                                outputs=[run_btn, loading_row, preview_row, example_input, example_labels, id2label_mapping_dataframe])
         
         run_btn.click(
             try_submit,
@@ -323,8 +318,6 @@ with gr.Blocks(theme=theme) as iface:
                 dataset_id_input,
                 dataset_config_input,
                 dataset_split_input,
-                column_mapping_input,
-                run_local,
             ],
             outputs=[
                 run_btn,

text_classification.py

@@ -72,10 +72,12 @@ def text_classification_fix_column_mapping(column_mapping, ppl, d_id, config, sp
     id2label_mapping = {}
     id2label = ppl.model.config.id2label
     label2id = {v: k for k, v in id2label.items()}
+    prediction_input = None
     prediction_result = None
     try:
         # Use the first item to test prediction
-        results = ppl({"text": df.head(1).at[0, column_mapping["text"]]}, top_k=None)
+        prediction_input = df.head(1).at[0, column_mapping["text"]]
+        results = ppl({"text": prediction_input}, top_k=None)
         prediction_result = {
             f'{result["label"]}({label2id[result["label"]]})': result["score"] for result in results
         }
@@ -85,33 +87,28 @@ def text_classification_fix_column_mapping(column_mapping, ppl, d_id, config, sp
 
     # Infer labels
     id2label_mapping, dataset_labels = text_classification_map_model_and_dataset_labels(id2label, dataset_features)
-    if "label" in column_mapping.keys():
-        if not isinstance(column_mapping["label"], dict) or set(column_mapping["label"].values()) != set(dataset_labels):
-            logging.warning(f'Provided {column_mapping["label"]} does not match labels in Dataset')
-            return column_mapping, prediction_result, None
-
-        if isinstance(column_mapping["label"], dict):
+    if "data" in column_mapping.keys():
+        if isinstance(column_mapping["data"], list):
             # Use the column mapping passed by user
-            for i, model_label in column_mapping["label"].items():
-                id2label_mapping[model_label] = dataset_labels[int(i)]
+            for user_label, model_label in column_mapping["data"]:
+                print(user_label, model_label)
+                id2label_mapping[model_label] = user_label
     elif None in id2label_mapping.values():
         column_mapping["label"] = {
             i: None for i in id2label.keys()
         }
         return column_mapping, prediction_result, None
 
-    id2label_mapping = {
-        v: k for k, v in id2label_mapping.items()
-    }
+    print(id2label_mapping)
     id2label_df = pd.DataFrame({
-        "ID": list(range(len(dataset_labels))),
-        "Labels": dataset_labels,
-        "Labels in original model": [f"{id2label_mapping[label]}({label2id[id2label_mapping[label]]})" for label in dataset_labels],
+        "Dataset Labels": dataset_labels,
+        "Model Prediction Labels": [id2label_mapping[label] for label in dataset_labels],
     })
-    if "label" not in column_mapping.keys():
+
+    if "data" not in column_mapping.keys():
         # Column mapping should contain original model labels
         column_mapping["label"] = {
             str(i): id2label_mapping[label] for i, label in zip(id2label.keys(), dataset_labels)
         }
 
-    return column_mapping, prediction_result, id2label_df
+    return column_mapping, prediction_input, prediction_result, id2label_df