URL: https://arxiv.org/pdf/2504.02768
%PDF-1.7
%����
1 0 obj
<< /Metadata 3 0 R /Names 4 0 R /OpenAction 5 0 R /PageMode /UseOutlines /Pages 6 0 R /Type /Catalog >>
endobj
2 0 obj
<< /Author (Jaap Jumelet; Leonie Weissweiler; Joakim Nivre; Arianna Bisazza) /Creator (arXiv GenPDF \(tex2pdf:a6404ea\)) /DOI (https://doi.org/10.48550/arXiv.2504.02768) /License (http://creativecommons.org/licenses/by/4.0/) /PTEX.Fullbanner (This is pdfTeX, Version 3.141592653-2.6-1.40.28 \(TeX Live 2025\) kpathsea version 6.4.1) /Producer (pikepdf 8.15.1) /Title (MultiBLiMP 1.0: A Massively Multilingual Benchmark of Linguistic Minimal Pairs) /Trapped /False /arXivID (https://arxiv.org/abs/2504.02768v4) >>
endobj
3 0 obj
<< /Subtype /XML /Type /Metadata /Length 1673 >>
stream
endstream
endobj
4 0 obj
<< /Dests 7 0 R >>
endobj
5 0 obj
<< /D [ 8 0 R /Fit ] /S /GoTo >>
endobj
6 0 obj
<< /Count 24 /Kids [ 9 0 R 10 0 R 11 0 R 12 0 R ] /Type /Pages >>
endobj
7 0 obj
<< /Kids [ 13 0 R 14 0 R 15 0 R 16 0 R 17 0 R ] /Limits [ (Doc-Start) (table.5) ] >>
endobj
8 0 obj
<< /Annots [ 18 0 R 19 0 R 20 0 R 21 0 R 22 0 R 23 0 R 24 0 R 25 0 R 26 0 R 27 0 R 28 0 R 29 0 R 30 0 R 31 0 R 32 0 R 33 0 R 34 0 R 35 0 R 36 0 R 37 0 R 38 0 R 39 0 R 40 0 R 41 0 R 42 0 R 43 0 R 44 0 R 45 0 R 46 0 R 47 0 R 48 0 R 49 0 R 50 0 R 51 0 R 52 0 R 53 0 R 54 0 R 55 0 R 56 0 R 57 0 R 58 0 R 59 0 R 60 0 R 61 0 R 62 0 R 63 0 R 64 0 R ] /Contents [ 65 0 R 66 0 R 67 0 R 68 0 R ] /MediaBox [ 0 0 595.276 841.89 ] /Parent 9 0 R /Resources 69 0 R /Type /Page >>
endobj
9 0 obj
<< /Count 6 /Kids [ 8 0 R 70 0 R 71 0 R 72 0 R 73 0 R 74 0 R ] /Parent 6 0 R /Type /Pages >>
endobj
10 0 obj
<< /Count 6 /Kids [ 75 0 R 76 0 R 77 0 R 78 0 R 79 0 R 80 0 R ] /Parent 6 0 R /Type /Pages >>
endobj
11 0 obj
<< /Count 6 /Kids [ 81 0 R 82 0 R 83 0 R 84 0 R 85 0 R 86 0 R ] /Parent 6 0 R /Type /Pages >>
endobj
12 0 obj
<< /Count 6 /Kids [ 87 0 R 88 0 R 89 0 R 90 0 R 91 0 R 92 0 R ] /Parent 6 0 R /Type /Pages >>
endobj
13 0 obj
<< /Kids [ 93 0 R 94 0 R 95 0 R 96 0 R 97 0 R 98 0 R ] /Limits [ (Doc-Start) (cite.glotcc) ] >>
endobj
14 0 obj
<< /Kids [ 99 0 R 100 0 R 101 0 R 102 0 R 103 0 R 104 0 R ] /Limits [ (cite.goldfish) (cite.wendler-etal-2024-llamas) ] >>
endobj
15 0 obj
<< /Kids [ 105 0 R 106 0 R 107 0 R 108 0 R 109 0 R 110 0 R ] /Limits [ (cite.wolf-etal-2020-transformers) (page.8) ] >>
endobj
16 0 obj
<< /Kids [ 111 0 R 112 0 R 113 0 R 114 0 R 115 0 R 116 0 R ] /Limits [ (page.9) (subsection.6.2) ] >>
endobj
17 0 obj
<< /Kids [ 117 0 R 118 0 R 119 0 R ] /Limits [ (subsection.6.3) (table.5) ] >>
endobj
18 0 obj
<< /A << /D (Hfootnote.1) /S /GoTo >> /Border [ 0 0 0 ] /C [ 1 0 0 ] /H /I /Rect [ 189.399 456.593 195.376 469.086 ] /Subtype /Link /Type /Annot >>
endobj
19 0 obj
<< /A << /D (cite.llama3) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 74.266 363.126 145.466 376.262 ] /Subtype /Link /Type /Annot >>
endobj
20 0 obj
<< /A << /D (cite.llama3) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 148.929 363.126 172.674 376.262 ] /Subtype /Link /Type /Annot >>
endobj
21 0 obj
<< /A << /D (cite.ustun-etal-2024-aya) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 176.442 363.126 228.316 376.262 ] /Subtype /Link /Type /Annot >>
endobj
22 0 obj
<< /A << /D (cite.ustun-etal-2024-aya) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 231.779 363.126 255.524 376.262 ] /Subtype /Link /Type /Annot >>
endobj
23 0 obj
<< /A << /D (cite.singh-etal-2024-aya) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 172.637 308.52 227.758 320.27 ] /Subtype /Link /Type /Annot >>
endobj
24 0 obj
<< /A << /D (cite.singh-etal-2024-aya) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 232.702 308.52 256.949 320.27 ] /Subtype /Link /Type /Annot >>
endobj
25 0 obj
<< /A << /D (cite.DBLP:journals/corr/abs-2412-03304) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 261.893 308.52 286.14 320.27 ] /Subtype /Link /Type /Annot >>
endobj
26 0 obj
<< /A << /D (cite.MAHOWALD2024517) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 242.062 267.872 291.264 279.623 ] /Subtype /Link /Type /Annot >>
endobj
27 0 obj
<< /A << /D (cite.MAHOWALD2024517) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 71.004 254.323 95.49 266.074 ] /Subtype /Link /Type /Annot >>
endobj
28 0 obj
<< /A << /D (cite.MAHOWALD2024517) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 99.924 254.323 124.171 266.074 ] /Subtype /Link /Type /Annot >>
endobj
29 0 obj
<< /A << /D (cite.goldfish) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 123.61 227.225 181.392 238.975 ] /Subtype /Link /Type /Annot >>
endobj
30 0 obj
<< /A << /D (cite.goldfish) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 186.125 227.225 215.935 238.975 ] /Subtype /Link /Type /Annot >>
endobj
31 0 obj
<< /A << /D (cite.dang2024aya) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 206.901 200.126 258.381 211.877 ] /Subtype /Link /Type /Annot >>
endobj
32 0 obj
<< /A << /D (cite.dang2024aya) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 262.439 200.126 286.686 211.877 ] /Subtype /Link /Type /Annot >>
endobj
33 0 obj
<< /A << /S /URI /Type /Action /URI (https://github.com/jumelet/multiblimp) >> /Border [ 0 0 0 ] /C [ 0 1 1 ] /H /I /Rect [ 111.343 87.327 243.348 98.741 ] /Subtype /Link /Type /Annot >>
endobj
34 0 obj
<< /A << /S /URI /Type /Action /URI (https://huggingface.co/datasets/jumelet/multiblimp) >> /Border [ 0 0 0 ] /C [ 0 1 1 ] /H /I /Rect [ 93.204 74.64 283.491 86.04 ] /Subtype /Link /Type /Annot >>
endobj
35 0 obj
<< /A << /D (cite.linzen-etal-2016-assessing) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 379.066 609.483 433.363 621.234 ] /Subtype /Link /Type /Annot >>
endobj
36 0 obj
<< /A << /D (cite.linzen-etal-2016-assessing) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 436.295 609.483 459.67 621.234 ] /Subtype /Link /Type /Annot >>
endobj
37 0 obj
<< /A << /D (cite.warstadt-etal-2020-blimp-benchmark) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 462.902 609.483 525.23 621.234 ] /Subtype /Link /Type /Annot >>
endobj
38 0 obj
<< /A << /D (cite.warstadt-etal-2020-blimp-benchmark) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 306.279 595.934 329.654 607.685 ] /Subtype /Link /Type /Annot >>
endobj
39 0 obj
<< /A << /D (cite.gulordava-etal-2018-colorless) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 309.479 542.146 379.49 553.488 ] /Subtype /Link /Type /Annot >>
endobj
40 0 obj
<< /A << /D (cite.gulordava-etal-2018-colorless) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 382.807 542.146 406.182 553.488 ] /Subtype /Link /Type /Annot >>
endobj
41 0 obj
<< /A << /D (cite.mueller-etal-2020-cross) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 409.798 542.146 469.02 553.488 ] /Subtype /Link /Type /Annot >>
endobj
42 0 obj
<< /A << /D (cite.mueller-etal-2020-cross) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 472.337 542.146 495.712 553.488 ] /Subtype /Link /Type /Annot >>
endobj
43 0 obj
<< /A << /D (cite.taktasheva-etal-2024-rublimp) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 499.328 542.146 526.54 553.488 ] /Subtype /Link /Type /Annot >>
endobj
44 0 obj
<< /A << /D (cite.taktasheva-etal-2024-rublimp) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 306.279 528.597 356.5 539.939 ] /Subtype /Link /Type /Annot >>
endobj
45 0 obj
<< /A << /D (cite.taktasheva-etal-2024-rublimp) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 359.962 528.597 383.773 539.939 ] /Subtype /Link /Type /Annot >>
endobj
46 0 obj
<< /A << /D (cite.nivre-etal-2016-universal) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 331.486 417.918 383.416 429.26 ] /Subtype /Link /Type /Annot >>
endobj
47 0 obj
<< /A << /D (cite.nivre-etal-2016-universal) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 387.217 417.918 411.464 429.26 ] /Subtype /Link /Type /Annot >>
endobj
48 0 obj
<< /A << /D (cite.nivre-etal-2020-universal) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 415.265 417.918 439.512 429.26 ] /Subtype /Link /Type /Annot >>
endobj
49 0 obj
<< /A << /D (cite.de-marneffe-etal-2021-universal) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 443.625 417.918 525.121 429.26 ] /Subtype /Link /Type /Annot >>
endobj
50 0 obj
<< /A << /D (cite.de-marneffe-etal-2021-universal) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 306.279 403.96 329.654 415.71 ] /Subtype /Link /Type /Annot >>
endobj
51 0 obj
<< /A << /D (cite.batsuren-etal-2022-unimorph) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 402.367 403.96 465.149 415.71 ] /Subtype /Link /Type /Annot >>
endobj
52 0 obj
<< /A << /D (cite.batsuren-etal-2022-unimorph) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 468.166 403.96 491.541 415.71 ] /Subtype /Link /Type /Annot >>
endobj
53 0 obj
<< /A << /D (figure.1) /S /GoTo >> /Border [ 0 0 0 ] /C [ 1 0 0 ] /H /I /Rect [ 382.989 363.312 390.327 375.063 ] /Subtype /Link /Type /Annot >>
endobj
54 0 obj
<< /A << /D (Hfootnote.2) /S /GoTo >> /Border [ 0 0 0 ] /C [ 1 0 0 ] /H /I /Rect [ 443.033 333.929 449.508 347.645 ] /Subtype /Link /Type /Annot >>
endobj
55 0 obj
<< /A << /D (figure.7) /S /GoTo >> /Border [ 0 0 0 ] /C [ 1 0 0 ] /H /I /Rect [ 330.583 239.084 338.139 250.835 ] /Subtype /Link /Type /Annot >>
endobj
56 0 obj
<< /A << /D (cite.mueller-etal-2020-cross) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 434.939 155.504 494.689 167.254 ] /Subtype /Link /Type /Annot >>
endobj
57 0 obj
<< /A << /D (cite.mueller-etal-2020-cross) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 498.127 155.504 521.632 167.254 ] /Subtype /Link /Type /Annot >>
endobj
58 0 obj
<< /A << /D (cite.brinkmann2025largelanguagemodelsshare) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 309.508 128.405 383.751 140.156 ] /Subtype /Link /Type /Annot >>
endobj
59 0 obj
<< /A << /D (cite.brinkmann2025largelanguagemodelsshare) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 387.199 128.405 410.748 140.156 ] /Subtype /Link /Type /Annot >>
endobj
60 0 obj
<< /A << /D (cite.levshina2023we) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 309.552 114.856 375.198 126.607 ] /Subtype /Link /Type /Annot >>
endobj
61 0 obj
<< /A << /D (cite.levshina2023we) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 378.66 114.856 402.471 126.607 ] /Subtype /Link /Type /Annot >>
endobj
62 0 obj
<< /A << /D (cite.baylor-etal-2024-multilingual) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 406.238 114.856 461.859 126.607 ] /Subtype /Link /Type /Annot >>
endobj
63 0 obj
<< /A << /D (cite.baylor-etal-2024-multilingual) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 465.321 114.856 489.132 126.607 ] /Subtype /Link /Type /Annot >>
endobj
64 0 obj
<< /A << /S /URI /URI (https://arxiv.org/abs/2504.02768v4) >> /BS << /W 0 >> /NM (fitz-L0) /Rect [ 12 247.07501 32 594.815 ] /Subtype /Link >>
endobj
65 0 obj
<< /Length 10 /Filter /FlateDecode >>
stream
x�+��|
endstream
endobj
66 0 obj
<< /Filter /FlateDecode /Length 4336 >>
stream
xڵZK�������a�Iо$'��f��|��@I�DE�|x<��O7�AQ��;Nj�������݈�]߾�.��y�'iT ��&Q��m�L"�Di ����M�c�0T}��|��b��(�"�j�� ��dBe
�� ��=5zȻ�\�(���3Wc_UY��wE�� ���~6[z�c����5X����w�3/�����"�c�\OK�D�3��_2��H_á����`�Tdi$��$��t�?��a� ��vw<���˂�?,�8,ʮ{*ʪh�Ɠ�V�����܀nAM�X ��_i�ۅ�a17dj�M�o1�7T�}[�u��a�]��c>�H����ծ�9E��i,L*N/v�~z:���B�(e��mW���r�m GQ�zvN "Qdg��~���? ���e�L��b� }R ���5YJ
����I<������w��u[ GQ��ʉ�x:I��O�E]���� *Ҷ��0���r��C��p&W�u�i,�E���k���|����NVK�2x2ȴ�q��T+�{��~X�Y�alX�}�l�5�DHX�pD������Є��84AɁf�4��6�3 B�i����L����L;��j�5X(��RD
