Update README.md
Browse filesAdded drop scores
README.md
CHANGED
|
@@ -191,7 +191,7 @@ By redesigning a common household item like the plastic bottle, we can create a
|
|
| 191 |
<th style="text-align:center; background-color: #001d6c; color: white;">PopQA</th>
|
| 192 |
<th style="text-align:center; background-color: #001d6c; color: white;">TruthfulQA</th>
|
| 193 |
<!-- <th style="text-align:center; background-color: #001d6c; color: white;">BigBenchHard</th> -->
|
| 194 |
-
|
| 195 |
<th style="text-align:center; background-color: #001d6c; color: white;">GSM8K</th>
|
| 196 |
<th style="text-align:center; background-color: #001d6c; color: white;">HumanEval</th>
|
| 197 |
<th style="text-align:center; background-color: #001d6c; color: white;">HumanEval+</th>
|
|
@@ -207,7 +207,7 @@ By redesigning a common household item like the plastic bottle, we can create a
|
|
| 207 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">20.55</td>
|
| 208 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">59.79</td>
|
| 209 |
<!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">54.46</td> -->
|
| 210 |
-
|
| 211 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">67.55</td>
|
| 212 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">79.45</td>
|
| 213 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">75.26</td>
|
|
@@ -222,7 +222,7 @@ By redesigning a common household item like the plastic bottle, we can create a
|
|
| 222 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">20.56</td>
|
| 223 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">59.8</td>
|
| 224 |
<!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">52.27</td> -->
|
| 225 |
-
|
| 226 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">67.02</td>
|
| 227 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">80.13</td>
|
| 228 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">73.39</td>
|
|
@@ -237,7 +237,7 @@ By redesigning a common household item like the plastic bottle, we can create a
|
|
| 237 |
<td style="text-align:center; background-color: #DAE8FF; color: black;"> 18.4 </td>
|
| 238 |
<td style="text-align:center; background-color: #DAE8FF; color: black;"> 58.97 </td>
|
| 239 |
<!-- <td style="text-align:center; background-color: #DAE8FF; color: black;"> 52.51 </td> -->
|
| 240 |
-
|
| 241 |
<td style="text-align:center; background-color: #DAE8FF; color: black;"> 72.48 </td>
|
| 242 |
<td style="text-align:center; background-color: #DAE8FF; color: black;"> 80.51 </td>
|
| 243 |
<td style="text-align:center; background-color: #DAE8FF; color: black;"> 75.68 </td>
|
|
@@ -253,7 +253,7 @@ By redesigning a common household item like the plastic bottle, we can create a
|
|
| 253 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">28.79</td>
|
| 254 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">52.79</td>
|
| 255 |
<!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">72.66</td> -->
|
| 256 |
-
|
| 257 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">83.24</td>
|
| 258 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">85.32</td>
|
| 259 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">80.15</td>
|
|
@@ -269,7 +269,7 @@ By redesigning a common household item like the plastic bottle, we can create a
|
|
| 269 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">13.25</td>
|
| 270 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">47.43</td>
|
| 271 |
<!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">65.71</td> -->
|
| 272 |
-
|
| 273 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">72.18</td>
|
| 274 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">67.54</td>
|
| 275 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">62.91</td>
|
|
@@ -285,7 +285,7 @@ By redesigning a common household item like the plastic bottle, we can create a
|
|
| 285 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">18.12</td>
|
| 286 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">63.06</td>
|
| 287 |
<!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">70.40</td> -->
|
| 288 |
-
|
| 289 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">84.46</td>
|
| 290 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">93.35</td>
|
| 291 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">89.91</td>
|
|
@@ -301,7 +301,7 @@ By redesigning a common household item like the plastic bottle, we can create a
|
|
| 301 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">9.94</td>
|
| 302 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">47.14</td>
|
| 303 |
<!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">65.04</td> -->
|
| 304 |
-
|
| 305 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">78.47</td>
|
| 306 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">79.89</td>
|
| 307 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">78.43</td>
|
|
@@ -316,7 +316,7 @@ By redesigning a common household item like the plastic bottle, we can create a
|
|
| 316 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">28.7</td>
|
| 317 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">65.84</td>
|
| 318 |
<!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">68.55</td> -->
|
| 319 |
-
|
| 320 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">79.15</td>
|
| 321 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">89.63</td>
|
| 322 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">85.79</td>
|
|
@@ -332,7 +332,7 @@ By redesigning a common household item like the plastic bottle, we can create a
|
|
| 332 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">28.04</td>
|
| 333 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">66.92</td>
|
| 334 |
<!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">64.77</td> -->
|
| 335 |
-
|
| 336 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">81.65</td>
|
| 337 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">89.35</td>
|
| 338 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">85.72</td>
|
|
@@ -347,7 +347,7 @@ By redesigning a common household item like the plastic bottle, we can create a
|
|
| 347 |
<td style="text-align:center; background-color: #DAE8FF; color: black;"> 26.17 </td>
|
| 348 |
<td style="text-align:center; background-color: #DAE8FF; color: black;"> 66.86 </td>
|
| 349 |
<!-- <td style="text-align:center; background-color: #DAE8FF; color: black;"> 59.01 </td> -->
|
| 350 |
-
|
| 351 |
<td style="text-align:center; background-color: #DAE8FF; color: black;"> 80.89 </td>
|
| 352 |
<td style="text-align:center; background-color: #DAE8FF; color: black;"> 89.73 </td>
|
| 353 |
<td style="text-align:center; background-color: #DAE8FF; color: black;"> 86.09 </td>
|
|
@@ -414,6 +414,8 @@ Granite-3.3-8B-Instruct builds upon Granite-3.3-8B-Base, leveraging both permiss
|
|
| 414 |
- 💡 Learn about the latest Granite learning resources: https://ibm.biz/granite-learning-resources
|
| 415 |
|
| 416 |
<p><a href="#fnref1" title="Jump back to reference">[1]</a> Evaluated using <a href="https://github.com/allenai/olmes">OLMES</a> (except AttaQ and Arena-Hard scores)</p>
|
|
|
|
|
|
|
| 417 |
<!-- ## Citation
|
| 418 |
```
|
| 419 |
@misc{granite-models,
|
|
|
|
| 191 |
<th style="text-align:center; background-color: #001d6c; color: white;">PopQA</th>
|
| 192 |
<th style="text-align:center; background-color: #001d6c; color: white;">TruthfulQA</th>
|
| 193 |
<!-- <th style="text-align:center; background-color: #001d6c; color: white;">BigBenchHard</th> -->
|
| 194 |
+
<th style="text-align:center; background-color: #001d6c; color: white;">DROP<sup id="fnref2"><a href="#fn2">2</a></sup></th>
|
| 195 |
<th style="text-align:center; background-color: #001d6c; color: white;">GSM8K</th>
|
| 196 |
<th style="text-align:center; background-color: #001d6c; color: white;">HumanEval</th>
|
| 197 |
<th style="text-align:center; background-color: #001d6c; color: white;">HumanEval+</th>
|
|
|
|
| 207 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">20.55</td>
|
| 208 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">59.79</td>
|
| 209 |
<!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">54.46</td> -->
|
| 210 |
+
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">20.99</td>
|
| 211 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">67.55</td>
|
| 212 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">79.45</td>
|
| 213 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">75.26</td>
|
|
|
|
| 222 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">20.56</td>
|
| 223 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">59.8</td>
|
| 224 |
<!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">52.27</td> -->
|
| 225 |
+
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">23.84</td>
|
| 226 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">67.02</td>
|
| 227 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">80.13</td>
|
| 228 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">73.39</td>
|
|
|
|
| 237 |
<td style="text-align:center; background-color: #DAE8FF; color: black;"> 18.4 </td>
|
| 238 |
<td style="text-align:center; background-color: #DAE8FF; color: black;"> 58.97 </td>
|
| 239 |
<!-- <td style="text-align:center; background-color: #DAE8FF; color: black;"> 52.51 </td> -->
|
| 240 |
+
<td style="text-align:center; background-color: #DAE8FF; color: black;"> 44.33 </td>
|
| 241 |
<td style="text-align:center; background-color: #DAE8FF; color: black;"> 72.48 </td>
|
| 242 |
<td style="text-align:center; background-color: #DAE8FF; color: black;"> 80.51 </td>
|
| 243 |
<td style="text-align:center; background-color: #DAE8FF; color: black;"> 75.68 </td>
|
|
|
|
| 253 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">28.79</td>
|
| 254 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">52.79</td>
|
| 255 |
<!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">72.66</td> -->
|
| 256 |
+
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">71.23</td>
|
| 257 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">83.24</td>
|
| 258 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">85.32</td>
|
| 259 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">80.15</td>
|
|
|
|
| 269 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">13.25</td>
|
| 270 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">47.43</td>
|
| 271 |
<!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">65.71</td> -->
|
| 272 |
+
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">49.73</td>
|
| 273 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">72.18</td>
|
| 274 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">67.54</td>
|
| 275 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">62.91</td>
|
|
|
|
| 285 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">18.12</td>
|
| 286 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">63.06</td>
|
| 287 |
<!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">70.40</td> -->
|
| 288 |
+
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">64.06</td>
|
| 289 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">84.46</td>
|
| 290 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">93.35</td>
|
| 291 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">89.91</td>
|
|
|
|
| 301 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">9.94</td>
|
| 302 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">47.14</td>
|
| 303 |
<!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">65.04</td> -->
|
| 304 |
+
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">51.78</td>
|
| 305 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">78.47</td>
|
| 306 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">79.89</td>
|
| 307 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">78.43</td>
|
|
|
|
| 316 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">28.7</td>
|
| 317 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">65.84</td>
|
| 318 |
<!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">68.55</td> -->
|
| 319 |
+
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">58.57</td>
|
| 320 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">79.15</td>
|
| 321 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">89.63</td>
|
| 322 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">85.79</td>
|
|
|
|
| 332 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">28.04</td>
|
| 333 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">66.92</td>
|
| 334 |
<!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">64.77</td> -->
|
| 335 |
+
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">58.29</td>
|
| 336 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">81.65</td>
|
| 337 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">89.35</td>
|
| 338 |
<td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">85.72</td>
|
|
|
|
| 347 |
<td style="text-align:center; background-color: #DAE8FF; color: black;"> 26.17 </td>
|
| 348 |
<td style="text-align:center; background-color: #DAE8FF; color: black;"> 66.86 </td>
|
| 349 |
<!-- <td style="text-align:center; background-color: #DAE8FF; color: black;"> 59.01 </td> -->
|
| 350 |
+
<td style="text-align:center; background-color: #DAE8FF; color: black;"> 59.36 </td>
|
| 351 |
<td style="text-align:center; background-color: #DAE8FF; color: black;"> 80.89 </td>
|
| 352 |
<td style="text-align:center; background-color: #DAE8FF; color: black;"> 89.73 </td>
|
| 353 |
<td style="text-align:center; background-color: #DAE8FF; color: black;"> 86.09 </td>
|
|
|
|
| 414 |
- 💡 Learn about the latest Granite learning resources: https://ibm.biz/granite-learning-resources
|
| 415 |
|
| 416 |
<p><a href="#fnref1" title="Jump back to reference">[1]</a> Evaluated using <a href="https://github.com/allenai/olmes">OLMES</a> (except AttaQ and Arena-Hard scores)</p>
|
| 417 |
+
<p><a href="#fnref2" title="Jump back to reference">[2]</a> Modified the implementation to handle some of the issues mentioned <a href="https://huggingface.co/blog/open-llm-leaderboard-drop">here</a></p>
|
| 418 |
+
<!-- ## Citation
|
| 419 |
<!-- ## Citation
|
| 420 |
```
|
| 421 |
@misc{granite-models,
|