sbert based roles matching

app.py

@@ -0,0 +1,99 @@
+# imports
+import json
+import time
+
+import gradio as gr
+from transformers import AutoTokenizer, AutoModel
+import openai
+# pytorch library
+import torch
+import torch.nn.functional as f
+
+from fuzzywuzzy import process
+
+from roles_list import roles
+from openai import OpenAI
+# Load the model from the specified directory
+embed_store = {}
+model = 'sentence-transformers/all-MiniLM-L12-v2'
+sbert_model = AutoModel.from_pretrained(model)
+sbert_tokenizer = AutoTokenizer.from_pretrained(model)
+
+client = OpenAI(
+    # defaults to os.environ.get("OPENAI_API_KEY")
+    api_key="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",
+)
+
+
+for role in roles:
+    encoding = sbert_tokenizer(role,  # the texts to be tokenized
+                               max_length=10,
+                               padding="max_length",
+                               return_tensors='pt'  # return the tensors (not lists)
+                               )
+    with torch.no_grad():
+        # get the model embeddings
+        embed = sbert_model(**encoding)
+        embed = embed.pooler_output
+    embed_store[role] = f.normalize(embed, p=2, dim=1)
+print("Model is ready for inference")
+
+
+def get_role_from_sbert(title):
+    start_time = time.time()
+    encoding = sbert_tokenizer(title,
+                         max_length=10,
+                         padding="max_length",
+                         return_tensors='pt'
+                         )
+    # Run the model prediction on the input data
+    with torch.no_grad():
+        # get the model embeddings
+        embed = sbert_model(**encoding)
+        embed = embed.pooler_output
+    store_cos = {}
+    for role in embed_store:
+        cos_sim = torch.nn.functional.cosine_similarity(f.normalize(embed, p=2, dim=1), embed_store[role])
+        store_cos[role] = round(cos_sim.item(), 3)
+    # Get the top 3 items with the highest cosine similarity
+    top_3_keys_values = sorted(store_cos.items(), key=lambda item: item[1], reverse=True)
+    job_scores_str = '\n'.join([f"{job}: {score}" for job, score in top_3_keys_values])
+
+    end_time = time.time()
+    execution_time = end_time - start_time
+    # Convert to dictionary if needed or keep as list of tuples
+    return job_scores_str + f" \nExecution time: {str(execution_time)}"
+
+
+def fuzzy_match(title):
+    """
+    Find the best matches for a query from a list of choices using fuzzy matching.
+
+    Parameters:
+    - query: The search string.
+    - choices: A list of strings to search through.
+    - limit: The maximum number of matches to return.
+
+    Returns:
+    A list of tuples with the match and its score. Higher score means closer match.
+    """
+    matches = process.extract(title, roles, limit=3)
+    return matches
+
+
+def fuzzy_match_sbert(title):
+    matches = fuzzy_match(title)
+    sbert_results = get_role_from_sbert(title)
+
+    new_list = [matches, sbert_results]
+    return new_list
+
+
+demo = gr.Interface(fn=get_role_from_sbert,
+                    inputs=gr.Textbox(label="Job Title"),
+                    outputs=gr.Textbox(label="Role"),
+                    title="HackerRank Role Classifier")
+
+
+gr.close_all()
+demo.launch(server_name='0.0.0.0', server_port=8081, share=True)

requirements.txt

@@ -0,0 +1,6 @@
+torch==1.13.1
+torchvision==0.14.1
+transformers==4.26.1
+gradio==3.18.0
+openai
+

roles_list.py

@@ -0,0 +1,71 @@
+roles = [
+    'Machine Learning Engineer',
+    'Sr. Machine Learning Engineer',
+    'Cloud Engineer',
+    'Cloud Engineer (AWS)',
+    'Cloud Security Engineer',
+    'Site Reliability Engineer',
+    'Sr. Cloud Engineer',
+    'Sr. Cloud Engineer (AWS)',
+    'Sr. Cloud Security Engineer',
+    'Sr. Site Reliability Engineer',
+    'Cybersecurity Engineer',
+    'Sr. Cybersecurity Engineer',
+    'Data Engineer',
+    'Data Engineer (Java Spark)',
+    'Data Engineer (PySpark)',
+    'Data Engineer (Scala Spark)',
+    'Sr. Data Engineer',
+    'Sr. Data Engineer (Java Spark)',
+    'Sr. Data Engineer (PySpark)',
+    'Sr. Data Engineer (Scala Spark)',
+    'Data Analyst',
+    'Data Analyst (Python)',
+    'Data Analyst (R)',
+    'Data Scientist',
+    'Sr. Data Analyst',
+    'Sr. Data Analyst (Python)',
+    'Sr. Data Analyst (R)',
+    'Sr. Data Scientist',
+    'Mobile Applications Developer (Android - Java)',
+    'Mobile Applications Developer (Android - Kotlin)',
+    'Mobile Applications Developer (React Native)',
+    'Sr. Mobile Applications Developer (Android - Java)',
+    'Sr. Mobile Applications Developer (Android - Kotlin)',
+    'Sr. Mobile Applications Developer (React Native)',
+    'QA Engineer (Selenium)',
+    'Quality Assurance Engineer',
+    'Quality Assurance Engineer (Mobile)',
+    'Sr. QA Engineer (Selenium)',
+    'Sr. Quality Assurance Engineer',
+    'Sr. Quality Assurance Engineer (Mobile)',
+    'Software Engineer',
+    'Software Engineer Intern',
+    'Sr. Software Engineer',
+    'Back-End Developer',
+    'Back-End Developer (.NET)',
+    'Back-End Developer (Django)',
+    'Back-End Developer (Laravel)',
+    'Back-End Developer (Node)',
+    'Back-End Developer (Rails)',
+    'Back-End Developer (Spring Boot)',
+    'Front-End Developer',
+    'Front-End Developer (Angular)',
+    'Front-End Developer (React)',
+    'Front-End Developer (Vue.js)',
+    'Full-Stack Engineer (Angular & Node)',
+    'Full-Stack Engineer (React & Node)',
+    'Sr. Back-End Developer',
+    'Sr. Back-End Developer (.NET)',
+    'Sr. Back-End Developer (Django)',
+    'Sr. Back-End Developer (Laravel)',
+    'Sr. Back-End Developer (Node)',
+    'Sr. Back-End Developer (Rails)',
+    'Sr. Back-End Developer (Spring Boot)',
+    'Sr. Front-End Developer',
+    'Sr. Front-End Developer (Angular)',
+    'Sr. Front-End Developer (React)',
+    'Sr. Front-End Developer (Vue.js)',
+    'Sr. Full-Stack Engineer (Angular & Node)',
+    'Sr. Full-Stack Engineer (React & Node)'
+]