setup cache

app.py

@@ -9,12 +9,47 @@ import tempfile
 from datetime import datetime, timedelta
 from functools import lru_cache
 from pathlib import Path
-
+import os, json, tempfile
+from pathlib import Path
+from huggingface_hub import hf_hub_download
 import gradio as gr
 
 # —— refactored helpers ——
 from modular_graph_and_candidates import build_graph_json, generate_html, build_timeline_json, generate_timeline_html
 
+def _escape_srcdoc(text: str) -> str:
+    """Escape for inclusion inside an <iframe srcdoc="…"> attribute."""
+    return (
+        text.replace("&", "&amp;")
+            .replace("\"", "&quot;")
+            .replace("'", "&#x27;")
+            .replace("<", "&lt;")
+            .replace(">", "&gt;")
+    )
+
+def _fetch_from_cache_repo(kind: str, sim_method: str, threshold: float, multimodal: bool):
+    """Fetch cached data from Molbap/hf_cached_embeds_log repo."""
+
+    repo_id = "Molbap/hf_cached_embeds_log"
+    try:
+        latest_fp = hf_hub_download(repo_id=repo_id, filename="latest.json")
+        info = json.loads(Path(latest_fp).read_text(encoding="utf-8"))
+        sha = info.get("sha")
+        key = f"{sha}/{sim_method}-{threshold:.2f}-m{int(multimodal)}"
+
+        html_fp = hf_hub_download(repo_id=repo_id, filename=f"{kind}/{key}.html")
+        json_fp = hf_hub_download(repo_id=repo_id, filename=f"{kind}/{key}.json")
+
+        raw_html = Path(html_fp).read_text(encoding="utf-8")
+        json_text = Path(json_fp).read_text(encoding="utf-8")
+
+        iframe_html = f'<iframe style="width:100%;height:85vh;border:none;" srcdoc="{_escape_srcdoc(raw_html)}"></iframe>'
+        tmp = Path(tempfile.mkstemp(suffix=("_timeline.json" if kind == "timeline" else ".json"))[1])
+        tmp.write_text(json_text, encoding="utf-8")
+        return iframe_html, str(tmp)
+    except Exception:
+        return None
+
 HF_MAIN_REPO = "https://github.com/huggingface/transformers"
 
 # ───────────────────────────── cache repo once per 24 h ───────────────────────────
@@ -40,19 +75,13 @@ def clone_or_cache(repo_url: str) -> Path:
 
 # ───────────────────────────── main callback ─────────────────────────────────────
 
-def _escape_srcdoc(text: str) -> str:
-    """Escape for inclusion inside an <iframe srcdoc="…"> attribute."""
-    return (
-        text.replace("&", "&amp;")
-            .replace("\"", "&quot;")
-            .replace("'", "&#x27;")
-            .replace("<", "&lt;")
-            .replace(">", "&gt;")
-    )
-
 
 def run_graph(repo_url: str, threshold: float, multimodal: bool, sim_method: str):
     """Generate the dependency graph visualization."""
+    hit = _fetch_from_cache_repo("graph", sim_method, threshold, multimodal)
+    if hit:
+        return hit
+
     repo_path = clone_or_cache(repo_url)
 
     graph = build_graph_json(
@@ -75,6 +104,10 @@ def run_graph(repo_url: str, threshold: float, multimodal: bool, sim_method: str
 
 def run_timeline(repo_url: str, threshold: float, multimodal: bool, sim_method: str):
     """Generate the chronological timeline visualization."""
+    hit = _fetch_from_cache_repo("timeline", sim_method, threshold, multimodal)
+    if hit:
+        return hit
+
     repo_path = clone_or_cache(repo_url)
 
     timeline = build_timeline_json(

build_cache.py

@@ -0,0 +1,107 @@
+import os
+import json
+import subprocess
+import tempfile
+from pathlib import Path
+from datetime import datetime, timezone
+from huggingface_hub import HfApi
+
+from modular_graph_and_candidates import (
+    build_graph_json,
+    generate_html,
+    build_timeline_json,
+    generate_timeline_html
+)
+
+REPO_URL = os.getenv("REPO_URL", "https://github.com/huggingface/transformers")
+CACHE_REPO = "Molbap/hf_cached_embeds_log"
+THRESH = float(os.getenv("SIM_THRESHOLD", "0.50"))
+MULTIMODAL = os.getenv("MULTIMODAL", "0") in {"1", "true", "True", "YES", "yes"}
+SIM_METHOD = os.getenv("SIM_METHOD", "jaccard")
+
+def main():
+    print(f"Building cache for {REPO_URL}")
+    print(f"Config: threshold={THRESH}, multimodal={MULTIMODAL}, method={SIM_METHOD}")
+
+    tmp = Path(tempfile.mkdtemp())
+    print(f"Working in {tmp}")
+
+    print("Cloning repository...")
+    subprocess.check_call([
+        "git", "clone", "--depth", "1", REPO_URL, str(tmp / "repo")
+    ])
+
+    sha = subprocess.check_output([
+        "git", "rev-parse", "HEAD"
+    ], cwd=tmp / "repo", text=True).strip()
+
+    print(f"Repository SHA: {sha}")
+
+    repo_path = tmp / "repo"
+
+    print("Building graph...")
+    graph = build_graph_json(
+        transformers_dir=repo_path,
+        threshold=THRESH,
+        multimodal=MULTIMODAL,
+        sim_method=SIM_METHOD,
+    )
+
+    print("Building timeline...")
+    timeline = build_timeline_json(
+        transformers_dir=repo_path,
+        threshold=THRESH,
+        multimodal=MULTIMODAL,
+        sim_method=SIM_METHOD,
+    )
+
+    print("Generating HTML...")
+    graph_html = generate_html(graph)
+    timeline_html = generate_timeline_html(timeline)
+
+    print(f"Uploading to {CACHE_REPO}...")
+
+    api = HfApi()
+
+    key = f"{sha}/{SIM_METHOD}-{THRESH:.2f}-m{int(MULTIMODAL)}"
+
+    latest = {
+        "sha": sha,
+        "updated_utc": datetime.now(timezone.utc).isoformat(),
+        "defaults": {
+            "sim_method": SIM_METHOD,
+            "threshold": THRESH,
+            "multimodal": MULTIMODAL
+        },
+        "paths": {
+            "graph_json": f"graph/{key}.json",
+            "graph_html": f"graph/{key}.html",
+            "timeline_json": f"timeline/{key}.json",
+            "timeline_html": f"timeline/{key}.html",
+        },
+    }
+
+    files_to_upload = [
+        (f"graph/{key}.json", json.dumps(graph, separators=(',', ':'))),
+        (f"graph/{key}.html", graph_html),
+        (f"timeline/{key}.json", json.dumps(timeline, separators=(',', ':'))),
+        (f"timeline/{key}.html", timeline_html),
+        ("latest.json", json.dumps(latest, separators=(',', ':'))),
+    ]
+
+    for path_in_repo, content in files_to_upload:
+        temp_file = tmp / "upload_temp"
+        temp_file.write_text(content, encoding="utf-8")
+
+        api.upload_file(
+            path_or_fileobj=str(temp_file),
+            path_in_repo=path_in_repo,
+            repo_id=CACHE_REPO,
+            commit_message=f"Cache update {sha[:7]} - {SIM_METHOD} t={THRESH} m={int(MULTIMODAL)}"
+        )
+        print(f"Uploaded {path_in_repo}")
+
+    print(f"Successfully uploaded cache for {key}")
+
+if __name__ == "__main__":
+    main()

modular_graph_and_candidates.py

@@ -685,7 +685,6 @@ function updateVisibility() {
 }
 document.getElementById('toggleRed').addEventListener('change', updateVisibility);
 
-const HF_LOGO_URI = "./static/hf-logo.png";
 const graph = __GRAPH_DATA__;
 const W = innerWidth, H = innerHeight;
 const svg = d3.select('#dependency').call(d3.zoom().on('zoom', e => g.attr('transform', e.transform)));
@@ -709,23 +708,7 @@ const node = g.selectAll('g.node')
   .call(d3.drag().on('start', dragStart).on('drag', dragged).on('end', dragEnd));
 
 const baseSel = node.filter(d => d.cls === 'base');
-if (HF_LOGO_URI){
-  baseSel.append('image')
-    .attr('href', HF_LOGO_URI)
-    .attr('width', 40)
-    .attr('height', 40)
-    .attr('x', -20)
-    .attr('y', -20)
-    .on('error', function() {
-      console.log('Image failed to load:', HF_LOGO_URI);
-      // Fallback to circle
-      d3.select(this.parentNode).append('circle')
-        .attr('r', 22).attr('fill', '#ffbe0b');
-    });
-  console.log('Loading logo from:', HF_LOGO_URI);
-}else{
-  baseSel.append('circle').attr('r', d => 22*d.sz).attr('fill', '#ffbe0b');
-}
+baseSel.append('circle').attr('r', d => 22*d.sz).attr('fill', '#ffbe0b');
 node.filter(d => d.cls !== 'base').append('circle').attr('r', d => 20*d.sz);
 
 node.append('text')
@@ -922,7 +905,6 @@ function updateVisibility() {
 }
 document.getElementById('toggleRed').addEventListener('change', updateVisibility);
 
-const HF_LOGO_URI = "./static/hf-logo.png";
 const timeline = __TIMELINE_DATA__;
 const W = innerWidth, H = innerHeight;
 
@@ -1003,20 +985,7 @@ const node = g.selectAll('g.node')
   .call(d3.drag().on('start', dragStart).on('drag', dragged).on('end', dragEnd));
 
 const baseSel = node.filter(d => d.cls === 'base');
-if (HF_LOGO_URI) {
-  baseSel.append('image')
-    .attr('href', HF_LOGO_URI)
-    .attr('width', 35)
-    .attr('height', 35)
-    .attr('x', -17.5)
-    .attr('y', -17.5)
-    .on('error', function() {
-      d3.select(this.parentNode).append('circle')
-        .attr('r', 20).attr('fill', '#ffbe0b');
-    });
-} else {
-  baseSel.append('circle').attr('r', 20).attr('fill', '#ffbe0b');
-}
+baseSel.append('circle').attr('r', 20).attr('fill', '#ffbe0b');
 node.filter(d => d.cls !== 'base').append('circle').attr('r', 18);
 
 node.append('text')