add octocoder and octogeex

community_results/bigcode_octocoder_loubnabnl/bigcode_octocoder_loubnabnl.json

@@ -0,0 +1 @@
+{"results": [{"task": "multiple-swift", "pass@1": 0.18240506329113923}, {"task": "multiple-lua", "pass@1": 0.2255900621118012}, {"task": "multiple-rkt", "pass@1": 0.10608695652173915}, {"task": "multiple-js", "pass@1": 0.32795031055900625}, {"task": "multiple-d", "pass@1": 0.1334615384615385}, {"task": "multiple-r", "pass@1": 0.14385093167701862}, {"task": "multiple-cpp", "pass@1": 0.29316770186335417}, {"task": "multiple-rs", "pass@1": 0.24256410256410257}, {"task": "multiple-jl", "pass@1": 0.24503144654088063}, {"task": "multiple-php", "pass@1": 0.2675776397515528}, {"task": "humaneval", "pass@1": 0.355609756097561}, {"task": "multiple-java", "pass@1": 0.26025316455696207}], "meta": {"model": "bigcode/octocoder"}}

community_results/bigcode_octocoder_loubnabnl/metrics_octocoder/metrics_humaneval_octocoder.json

@@ -0,0 +1,11 @@
+{
+  "humaneval": {
+    "pass@1": 0.355609756097561,
+    "pass@10": 0.5180529150889583
+  },
+  "config": {
+    "model": "bigcode/octocoder",
+    "temperature": 0.2,
+    "n_samples": 50
+  }
+}

community_results/bigcode_octocoder_loubnabnl/metrics_octocoder/metrics_humanevalsynthesize-python_octocoder.json

@@ -0,0 +1,43 @@
+{
+  "humanevalsynthesize-python": {
+    "pass@1": 0.45304878048780495,
+    "pass@10": 0.6338166655659803
+  },
+  "config": {
+    "prefix": "",
+    "do_sample": true,
+    "temperature": 0.2,
+    "top_k": 0,
+    "top_p": 0.95,
+    "n_samples": 50,
+    "eos": "<|endoftext|>",
+    "seed": 0,
+    "model": "bigcode/octocoder",
+    "modeltype": "causal",
+    "peft_model": null,
+    "revision": null,
+    "use_auth_token": true,
+    "trust_remote_code": true,
+    "tasks": "humanevalsynthesize-python",
+    "instruction_tokens": null,
+    "batch_size": 50,
+    "max_length_generation": 2048,
+    "precision": "bf16",
+    "load_in_8bit": false,
+    "load_in_4bit": false,
+    "limit": null,
+    "limit_start": 0,
+    "postprocess": true,
+    "allow_code_execution": true,
+    "generation_only": false,
+    "load_generations_path": null,
+    "load_data_path": null,
+    "metric_output_path": "/fsx/loubna/code/dev/leader/bigcode-evaluation-harness/generations_octocoder/metrics24_humanevalsynthesize-python_octocoder.json",
+    "save_generations": true,
+    "save_generations_path": "/fsx/loubna/code/dev/leader/bigcode-evaluation-harness/generations_octocoder/generations24_humanevalsynthesize-python_octocoder.json",
+    "save_references": false,
+    "prompt": "octocoder",
+    "max_memory_per_gpu": null,
+    "check_references": false
+  }
+}

community_results/bigcode_octocoder_loubnabnl/metrics_octocoder/metrics_multiple-cpp_octocoder.json

@@ -0,0 +1,11 @@
+{
+  "multiple-cpp": {
+    "pass@1": 0.29316770186335417,
+    "pass@10": 0.418989804893579
+  },
+  "config": {
+    "model": "bigcode/octocoder",
+    "temperature": 0.2,
+    "n_samples": 50
+  }
+}

community_results/bigcode_octocoder_loubnabnl/metrics_octocoder/metrics_multiple-d_octocoder.json

@@ -0,0 +1,11 @@
+{
+  "multiple-d": {
+    "pass@1": 0.1334615384615385,
+    "pass@10": 0.22191295496585037
+  },
+  "config": {
+    "model": "bigcode/octocoder",
+    "temperature": 0.2,
+    "n_samples": 50
+  }
+}

community_results/bigcode_octocoder_loubnabnl/metrics_octocoder/metrics_multiple-java_octocoder.json

@@ -0,0 +1,11 @@
+{
+  "multiple-java": {
+    "pass@1": 0.26025316455696207,
+    "pass@10": 0.3844432224837278
+  },
+  "config": {
+    "model": "bigcode/octocoder",
+    "temperature": 0.2,
+    "n_samples": 50
+  }
+}

community_results/bigcode_octocoder_loubnabnl/metrics_octocoder/metrics_multiple-jl_octocoder.json

@@ -0,0 +1,11 @@
+{
+  "multiple-jl": {
+    "pass@1": 0.24503144654088063,
+    "pass@10": 0.37015780634005285
+  },
+  "config": {
+    "model": "bigcode/octocoder",
+    "temperature": 0.2,
+    "n_samples": 50
+  }
+}

community_results/bigcode_octocoder_loubnabnl/metrics_octocoder/metrics_multiple-js_octocoder.json

@@ -0,0 +1,11 @@
+{
+  "multiple-js": {
+    "pass@1": 0.32795031055900625,
+    "pass@10": 0.46965261958375676
+  },
+  "config": {
+    "model": "bigcode/octocoder",
+    "temperature": 0.2,
+    "n_samples": 50
+  }
+}

community_results/bigcode_octocoder_loubnabnl/metrics_octocoder/metrics_multiple-lua_octocoder.json

@@ -0,0 +1,11 @@
+{
+  "multiple-lua": {
+    "pass@1": 0.2255900621118012,
+    "pass@10": 0.3642531020086279
+  },
+  "config": {
+    "model": "bigcode/octocoder",
+    "temperature": 0.2,
+    "n_samples": 50
+  }
+}

community_results/bigcode_octocoder_loubnabnl/metrics_octocoder/metrics_multiple-php_octocoder.json

@@ -0,0 +1,11 @@
+{
+  "multiple-php": {
+    "pass@1": 0.2675776397515528,
+    "pass@10": 0.4406294303479089
+  },
+  "config": {
+    "model": "bigcode/octocoder",
+    "temperature": 0.2,
+    "n_samples": 50
+  }
+}

community_results/bigcode_octocoder_loubnabnl/metrics_octocoder/metrics_multiple-r_octocoder.json

@@ -0,0 +1,11 @@
+{
+  "multiple-r": {
+    "pass@1": 0.14385093167701862,
+    "pass@10": 0.23625313277669435
+  },
+  "config": {
+    "model": "bigcode/octocoder",
+    "temperature": 0.2,
+    "n_samples": 50
+  }
+}

community_results/bigcode_octocoder_loubnabnl/metrics_octocoder/metrics_multiple-rkt_octocoder.json

@@ -0,0 +1,11 @@
+{
+  "multiple-rkt": {
+    "pass@1": 0.10608695652173915,
+    "pass@10": 0.18583082734989625
+  },
+  "config": {
+    "model": "bigcode/octocoder",
+    "temperature": 0.2,
+    "n_samples": 50
+  }
+}

community_results/bigcode_octocoder_loubnabnl/metrics_octocoder/metrics_multiple-rs_octocoder.json

@@ -0,0 +1,11 @@
+{
+  "multiple-rs": {
+    "pass@1": 0.24256410256410257,
+    "pass@10": 0.37981269956491903
+  },
+  "config": {
+    "model": "bigcode/octocoder",
+    "temperature": 0.2,
+    "n_samples": 50
+  }
+}

community_results/bigcode_octocoder_loubnabnl/metrics_octocoder/metrics_multiple-swift_octocoder.json

@@ -0,0 +1,11 @@
+{
+  "multiple-swift": {
+    "pass@1": 0.18240506329113923,
+    "pass@10": 0.27015964860608316
+  },
+  "config": {
+    "model": "bigcode/octocoder",
+    "temperature": 0.2,
+    "n_samples": 50
+  }
+}

community_results/bigcode_octogeex_loubnabnl/bigcode_octogeex_loubnabnl.json

@@ -0,0 +1 @@
+{"results": [{"task": "humaneval", "pass@1": 0.3579268292682927}, {"task": "multiple-swift", "pass@1": 0.17025316455696202}, {"task": "multiple-cpp", "pass@1": 0.2392546583850932}, {"task": "multiple-js", "pass@1": 0.28496894409937884}, {"task": "multiple-lua", "pass@1": 0.1618633540372671}, {"task": "multiple-d", "pass@1": 0.09769230769230768}, {"task": "multiple-rs", "pass@1": 0.17935897435897438}, {"task": "multiple-r", "pass@1": 0.1366459627329193}, {"task": "multiple-php", "pass@1": 0.25850931677018635}, {"task": "multiple-rkt", "pass@1": 0.12024844720496894}, {"task": "multiple-jl", "pass@1": 0.22943396226415103}, {"task": "multiple-java", "pass@1": 0.1932911392405063}], "meta": {"model": "bigcode/octogeex"}}

community_results/bigcode_octogeex_loubnabnl/metrics_octogeex/metrics_humaneval_octogeex.json

@@ -0,0 +1,11 @@
+{
+  "humaneval": {
+    "pass@1": 0.3579268292682927,
+    "pass@10": 0.5255121713418224
+  },
+  "config": {
+    "model": "bigcode/octogeex",
+    "temperature": 0.2,
+    "n_samples": 50
+  }
+}

community_results/bigcode_octogeex_loubnabnl/metrics_octogeex/metrics_humanevalsynthesize-python_octogeex.json

@@ -0,0 +1,43 @@
+{
+  "humanevalsynthesize-python": {
+    "pass@1": 0.42280487804878053,
+    "pass@10": 0.615600414422801
+  },
+  "config": {
+    "prefix": "",
+    "do_sample": true,
+    "temperature": 0.2,
+    "top_k": 0,
+    "top_p": 0.95,
+    "n_samples": 50,
+    "eos": "<|endoftext|>",
+    "seed": 0,
+    "model": "bigcode/octogeex",
+    "modeltype": "causal",
+    "peft_model": null,
+    "revision": null,
+    "use_auth_token": true,
+    "trust_remote_code": true,
+    "tasks": "humanevalsynthesize-python",
+    "instruction_tokens": null,
+    "batch_size": 50,
+    "max_length_generation": 2048,
+    "precision": "bf16",
+    "load_in_8bit": false,
+    "load_in_4bit": false,
+    "limit": null,
+    "limit_start": 0,
+    "postprocess": true,
+    "allow_code_execution": true,
+    "generation_only": false,
+    "load_generations_path": null,
+    "load_data_path": null,
+    "metric_output_path": "/fsx/loubna/code/dev/leader/bigcode-evaluation-harness/generations_octogeex/metrics24_humanevalsynthesize-python_octogeex.json",
+    "save_generations": true,
+    "save_generations_path": "/fsx/loubna/code/dev/leader/bigcode-evaluation-harness/generations_octogeex/generations24_humanevalsynthesize-python_octogeex.json",
+    "save_references": false,
+    "prompt": "octogeex",
+    "max_memory_per_gpu": null,
+    "check_references": false
+  }
+}

community_results/bigcode_octogeex_loubnabnl/metrics_octogeex/metrics_multiple-cpp_octogeex.json

@@ -0,0 +1,11 @@
+{
+  "multiple-cpp": {
+    "pass@1": 0.2392546583850932,
+    "pass@10": 0.3770275481552431
+  },
+  "config": {
+    "model": "bigcode/octogeex",
+    "temperature": 0.2,
+    "n_samples": 50
+  }
+}

community_results/bigcode_octogeex_loubnabnl/metrics_octogeex/metrics_multiple-d_octogeex.json

@@ -0,0 +1,11 @@
+{
+  "multiple-d": {
+    "pass@1": 0.09769230769230768,
+    "pass@10": 0.17328015912854275
+  },
+  "config": {
+    "model": "bigcode/octogeex",
+    "temperature": 0.2,
+    "n_samples": 50
+  }
+}

community_results/bigcode_octogeex_loubnabnl/metrics_octogeex/metrics_multiple-java_octogeex.json

@@ -0,0 +1,11 @@
+{
+  "multiple-java": {
+    "pass@1": 0.1932911392405063,
+    "pass@10": 0.2956204163601109
+  },
+  "config": {
+    "model": "bigcode/octogeex",
+    "temperature": 0.2,
+    "n_samples": 50
+  }
+}

community_results/bigcode_octogeex_loubnabnl/metrics_octogeex/metrics_multiple-jl_octogeex.json

@@ -0,0 +1,11 @@
+{
+  "multiple-jl": {
+    "pass@1": 0.22943396226415103,
+    "pass@10": 0.31614256642955946
+  },
+  "config": {
+    "model": "bigcode/octogeex",
+    "temperature": 0.2,
+    "n_samples": 50
+  }
+}

community_results/bigcode_octogeex_loubnabnl/metrics_octogeex/metrics_multiple-js_octogeex.json

@@ -0,0 +1,11 @@
+{
+  "multiple-js": {
+    "pass@1": 0.28496894409937884,
+    "pass@10": 0.4542697803948776
+  },
+  "config": {
+    "model": "bigcode/octogeex",
+    "temperature": 0.2,
+    "n_samples": 50
+  }
+}

community_results/bigcode_octogeex_loubnabnl/metrics_octogeex/metrics_multiple-lua_octogeex.json

@@ -0,0 +1,11 @@
+{
+  "multiple-lua": {
+    "pass@1": 0.1618633540372671,
+    "pass@10": 0.25142068318547794
+  },
+  "config": {
+    "model": "bigcode/octogeex",
+    "temperature": 0.2,
+    "n_samples": 50
+  }
+}

community_results/bigcode_octogeex_loubnabnl/metrics_octogeex/metrics_multiple-php_octogeex.json

@@ -0,0 +1,11 @@
+{
+  "multiple-php": {
+    "pass@1": 0.25850931677018635,
+    "pass@10": 0.4130885963164481
+  },
+  "config": {
+    "model": "bigcode/octogeex",
+    "temperature": 0.2,
+    "n_samples": 50
+  }
+}