Update modules/syntax_analysis.py
Browse files- modules/syntax_analysis.py +63 -31
modules/syntax_analysis.py
CHANGED
|
@@ -5,12 +5,7 @@ import networkx as nx
|
|
| 5 |
import matplotlib.pyplot as plt
|
| 6 |
from collections import Counter
|
| 7 |
|
| 8 |
-
|
| 9 |
-
def load_spacy_model():
|
| 10 |
-
return spacy.load("es_core_news_lg")
|
| 11 |
-
|
| 12 |
-
# Load spaCy model
|
| 13 |
-
nlp = spacy.load("es_core_news_lg")
|
| 14 |
|
| 15 |
# Define colors for grammatical categories
|
| 16 |
POS_COLORS = {
|
|
@@ -33,28 +28,66 @@ POS_COLORS = {
|
|
| 33 |
}
|
| 34 |
|
| 35 |
POS_TRANSLATIONS = {
|
| 36 |
-
'
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
}
|
| 53 |
|
| 54 |
def count_pos(doc):
|
| 55 |
return Counter(token.pos_ for token in doc if token.pos_ != 'PUNCT')
|
| 56 |
|
| 57 |
-
def create_syntax_graph(doc):
|
| 58 |
G = nx.DiGraph()
|
| 59 |
pos_counts = count_pos(doc)
|
| 60 |
word_nodes = {}
|
|
@@ -69,7 +102,7 @@ def create_syntax_graph(doc):
|
|
| 69 |
color = POS_COLORS.get(token.pos_, '#FFFFFF')
|
| 70 |
word_colors[lower_text] = color
|
| 71 |
G.add_node(node_id,
|
| 72 |
-
label=f"{token.text}\n[{POS_TRANSLATIONS.get(token.pos_, token.pos_)}]",
|
| 73 |
pos=token.pos_,
|
| 74 |
size=pos_counts[token.pos_] * 500,
|
| 75 |
color=color)
|
|
@@ -81,8 +114,8 @@ def create_syntax_graph(doc):
|
|
| 81 |
|
| 82 |
return G, word_colors
|
| 83 |
|
| 84 |
-
def visualize_syntax_graph(doc):
|
| 85 |
-
G, word_colors = create_syntax_graph(doc)
|
| 86 |
|
| 87 |
plt.figure(figsize=(20, 15))
|
| 88 |
pos = nx.spring_layout(G, k=2, iterations=100)
|
|
@@ -97,24 +130,23 @@ def visualize_syntax_graph(doc):
|
|
| 97 |
edge_labels = nx.get_edge_attributes(G, 'label')
|
| 98 |
nx.draw_networkx_edge_labels(G, pos, edge_labels=edge_labels, font_size=8)
|
| 99 |
|
| 100 |
-
plt.title("An谩lisis Sint谩ctico")
|
| 101 |
plt.axis('off')
|
| 102 |
|
| 103 |
-
legend_elements = [plt.Rectangle((0,0),1,1, facecolor=color, edgecolor='none', label=f"{POS_TRANSLATIONS[pos]} ({count_pos(doc)[pos]})")
|
| 104 |
for pos, color in POS_COLORS.items() if pos in set(nx.get_node_attributes(G, 'pos').values())]
|
| 105 |
plt.legend(handles=legend_elements, loc='center left', bbox_to_anchor=(1, 0.5))
|
| 106 |
|
| 107 |
return plt
|
| 108 |
|
| 109 |
-
def visualize_syntax(text):
|
| 110 |
max_tokens = 5000
|
| 111 |
doc = nlp(text)
|
| 112 |
if len(doc) > max_tokens:
|
| 113 |
doc = nlp(text[:max_tokens])
|
| 114 |
print(f"Warning: The input text is too long. Only the first {max_tokens} tokens will be visualized.")
|
| 115 |
-
return visualize_syntax_graph(doc)
|
| 116 |
|
| 117 |
-
# Repeated words colors
|
| 118 |
def get_repeated_words_colors(doc):
|
| 119 |
word_counts = Counter(token.text.lower() for token in doc if token.pos_ != 'PUNCT')
|
| 120 |
repeated_words = {word: count for word, count in word_counts.items() if count > 1}
|
|
|
|
| 5 |
import matplotlib.pyplot as plt
|
| 6 |
from collections import Counter
|
| 7 |
|
| 8 |
+
# Remove the global nlp model loading
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
|
| 10 |
# Define colors for grammatical categories
|
| 11 |
POS_COLORS = {
|
|
|
|
| 28 |
}
|
| 29 |
|
| 30 |
POS_TRANSLATIONS = {
|
| 31 |
+
'es': {
|
| 32 |
+
'ADJ': 'Adjetivo',
|
| 33 |
+
'ADP': 'Adposici贸n',
|
| 34 |
+
'ADV': 'Adverbio',
|
| 35 |
+
'AUX': 'Auxiliar',
|
| 36 |
+
'CCONJ': 'Conjunci贸n Coordinante',
|
| 37 |
+
'DET': 'Determinante',
|
| 38 |
+
'INTJ': 'Interjecci贸n',
|
| 39 |
+
'NOUN': 'Sustantivo',
|
| 40 |
+
'NUM': 'N煤mero',
|
| 41 |
+
'PART': 'Part铆cula',
|
| 42 |
+
'PRON': 'Pronombre',
|
| 43 |
+
'PROPN': 'Nombre Propio',
|
| 44 |
+
'SCONJ': 'Conjunci贸n Subordinante',
|
| 45 |
+
'SYM': 'S铆mbolo',
|
| 46 |
+
'VERB': 'Verbo',
|
| 47 |
+
'X': 'Otro',
|
| 48 |
+
},
|
| 49 |
+
'en': {
|
| 50 |
+
'ADJ': 'Adjective',
|
| 51 |
+
'ADP': 'Adposition',
|
| 52 |
+
'ADV': 'Adverb',
|
| 53 |
+
'AUX': 'Auxiliary',
|
| 54 |
+
'CCONJ': 'Coordinating Conjunction',
|
| 55 |
+
'DET': 'Determiner',
|
| 56 |
+
'INTJ': 'Interjection',
|
| 57 |
+
'NOUN': 'Noun',
|
| 58 |
+
'NUM': 'Number',
|
| 59 |
+
'PART': 'Particle',
|
| 60 |
+
'PRON': 'Pronoun',
|
| 61 |
+
'PROPN': 'Proper Noun',
|
| 62 |
+
'SCONJ': 'Subordinating Conjunction',
|
| 63 |
+
'SYM': 'Symbol',
|
| 64 |
+
'VERB': 'Verb',
|
| 65 |
+
'X': 'Other',
|
| 66 |
+
},
|
| 67 |
+
'fr': {
|
| 68 |
+
'ADJ': 'Adjectif',
|
| 69 |
+
'ADP': 'Adposition',
|
| 70 |
+
'ADV': 'Adverbe',
|
| 71 |
+
'AUX': 'Auxiliaire',
|
| 72 |
+
'CCONJ': 'Conjonction de Coordination',
|
| 73 |
+
'DET': 'D茅terminant',
|
| 74 |
+
'INTJ': 'Interjection',
|
| 75 |
+
'NOUN': 'Nom',
|
| 76 |
+
'NUM': 'Nombre',
|
| 77 |
+
'PART': 'Particule',
|
| 78 |
+
'PRON': 'Pronom',
|
| 79 |
+
'PROPN': 'Nom Propre',
|
| 80 |
+
'SCONJ': 'Conjonction de Subordination',
|
| 81 |
+
'SYM': 'Symbole',
|
| 82 |
+
'VERB': 'Verbe',
|
| 83 |
+
'X': 'Autre',
|
| 84 |
+
}
|
| 85 |
}
|
| 86 |
|
| 87 |
def count_pos(doc):
|
| 88 |
return Counter(token.pos_ for token in doc if token.pos_ != 'PUNCT')
|
| 89 |
|
| 90 |
+
def create_syntax_graph(doc, lang):
|
| 91 |
G = nx.DiGraph()
|
| 92 |
pos_counts = count_pos(doc)
|
| 93 |
word_nodes = {}
|
|
|
|
| 102 |
color = POS_COLORS.get(token.pos_, '#FFFFFF')
|
| 103 |
word_colors[lower_text] = color
|
| 104 |
G.add_node(node_id,
|
| 105 |
+
label=f"{token.text}\n[{POS_TRANSLATIONS[lang].get(token.pos_, token.pos_)}]",
|
| 106 |
pos=token.pos_,
|
| 107 |
size=pos_counts[token.pos_] * 500,
|
| 108 |
color=color)
|
|
|
|
| 114 |
|
| 115 |
return G, word_colors
|
| 116 |
|
| 117 |
+
def visualize_syntax_graph(doc, lang):
|
| 118 |
+
G, word_colors = create_syntax_graph(doc, lang)
|
| 119 |
|
| 120 |
plt.figure(figsize=(20, 15))
|
| 121 |
pos = nx.spring_layout(G, k=2, iterations=100)
|
|
|
|
| 130 |
edge_labels = nx.get_edge_attributes(G, 'label')
|
| 131 |
nx.draw_networkx_edge_labels(G, pos, edge_labels=edge_labels, font_size=8)
|
| 132 |
|
| 133 |
+
plt.title("Syntactic Analysis" if lang == 'en' else "Analyse Syntaxique" if lang == 'fr' else "An谩lisis Sint谩ctico")
|
| 134 |
plt.axis('off')
|
| 135 |
|
| 136 |
+
legend_elements = [plt.Rectangle((0,0),1,1, facecolor=color, edgecolor='none', label=f"{POS_TRANSLATIONS[lang][pos]} ({count_pos(doc)[pos]})")
|
| 137 |
for pos, color in POS_COLORS.items() if pos in set(nx.get_node_attributes(G, 'pos').values())]
|
| 138 |
plt.legend(handles=legend_elements, loc='center left', bbox_to_anchor=(1, 0.5))
|
| 139 |
|
| 140 |
return plt
|
| 141 |
|
| 142 |
+
def visualize_syntax(text, nlp, lang):
|
| 143 |
max_tokens = 5000
|
| 144 |
doc = nlp(text)
|
| 145 |
if len(doc) > max_tokens:
|
| 146 |
doc = nlp(text[:max_tokens])
|
| 147 |
print(f"Warning: The input text is too long. Only the first {max_tokens} tokens will be visualized.")
|
| 148 |
+
return visualize_syntax_graph(doc, lang)
|
| 149 |
|
|
|
|
| 150 |
def get_repeated_words_colors(doc):
|
| 151 |
word_counts = Counter(token.text.lower() for token in doc if token.pos_ != 'PUNCT')
|
| 152 |
repeated_words = {word: count for word, count in word_counts.items() if count > 1}
|