{
  "name": "NoaLingua Language Data Index",
  "version": "1.0",
  "published": "2026-09-09",
  "measured": "2026-08-17",
  "url": "https://noalingua.com/research/language-data-index/",
  "license": "https://creativecommons.org/licenses/by/4.0/",
  "creator": "Nemanja Pejić",
  "publisher": "NoaSolutions",
  "citation": "Pejić, N. (2026). NoaLingua Language Data Index (v1.0) [Data set]. NoaSolutions. https://noalingua.com/research/language-data-index/",
  "method": "Availability by HTTP status probe per corpus release URL; pack sizes by HTTP HEAD; paradigm rows read from the downloaded files. Measured against live services.",
  "sources": [
    {
      "id": "morphology",
      "name": "Grammar tables",
      "corpus": "UniMorph",
      "url": "https://unimorph.github.io/",
      "license": "CC BY-SA 3.0",
      "provides": "Inflection paradigms — conjugation and declension, form by form."
    },
    {
      "id": "wiktionary",
      "name": "Dictionary and IPA",
      "corpus": "kaikki.org (Wiktionary extraction)",
      "url": "https://kaikki.org/",
      "license": "CC BY-SA 4.0",
      "provides": "Definitions, pronunciation and register labels."
    },
    {
      "id": "examples",
      "name": "Example sentences",
      "corpus": "Tatoeba",
      "url": "https://tatoeba.org/",
      "license": "CC BY 2.0 FR",
      "provides": "Human-written bilingual sentence pairs."
    },
    {
      "id": "frequencyList",
      "name": "Word frequency",
      "corpus": "Hermit Dave / OpenSubtitles 2018",
      "url": "https://github.com/hermitdave/FrequencyWords",
      "license": "MIT",
      "provides": "Ranked word lists, used to mark which words are uncommon."
    },
    {
      "id": "punctuationModel",
      "name": "Punctuation restoration",
      "corpus": "anchor-flux/punct-pcs-47lang",
      "url": "https://huggingface.co/",
      "license": "Apache 2.0",
      "provides": "Restores sentence boundaries in captions that arrive without any."
    }
  ],
  "summary": {
    "languages": 28,
    "sources": 5,
    "cells": 140,
    "filled_cells": 105,
    "complete_coverage": [
      "en",
      "es",
      "fr",
      "de",
      "it",
      "pt",
      "ru"
    ],
    "single_source_only": [
      "vi",
      "th"
    ],
    "dictionary_total_mb": 1416,
    "dictionary_measured": 19,
    "grammar_total_mb": 349,
    "grammar_measured": 24,
    "counted_paradigm_rows": 3433493,
    "per_source": [
      {
        "id": "morphology",
        "covered": 24,
        "share_percent": 86,
        "missing": [
          "Chinese",
          "Vietnamese",
          "Thai",
          "Finnish"
        ]
      },
      {
        "id": "wiktionary",
        "covered": 24,
        "share_percent": 86,
        "missing": [
          "Serbian",
          "Croatian",
          "Vietnamese",
          "Thai"
        ]
      },
      {
        "id": "examples",
        "covered": 28,
        "share_percent": 100,
        "missing": []
      },
      {
        "id": "frequencyList",
        "covered": 7,
        "share_percent": 25,
        "missing": [
          "Dutch",
          "Polish",
          "Swedish",
          "Japanese",
          "Chinese",
          "Korean",
          "Arabic",
          "Turkish",
          "Serbian",
          "Croatian",
          "Czech",
          "Greek",
          "Hindi",
          "Vietnamese",
          "Thai",
          "Ukrainian",
          "Romanian",
          "Hungarian",
          "Finnish",
          "Danish",
          "Norwegian"
        ]
      },
      {
        "id": "punctuationModel",
        "covered": 22,
        "share_percent": 79,
        "missing": [
          "Swedish",
          "Czech",
          "Vietnamese",
          "Thai",
          "Danish",
          "Norwegian"
        ]
      }
    ]
  },
  "rows": [
    {
      "code": "en",
      "language": "English",
      "native_name": "English",
      "script": "latin",
      "grammar_unimorph": true,
      "dictionary_kaikki": true,
      "examples_tatoeba": true,
      "frequency_list": true,
      "punctuation_model": true,
      "sources_covered": 5,
      "grammar_pack_mb": 18,
      "dictionary_pack_mb": 475,
      "grammar_paradigm_rows": null,
      "grammar_verbs_only": false
    },
    {
      "code": "es",
      "language": "Spanish",
      "native_name": "Español",
      "script": "latin",
      "grammar_unimorph": true,
      "dictionary_kaikki": true,
      "examples_tatoeba": true,
      "frequency_list": true,
      "punctuation_model": true,
      "sources_covered": 5,
      "grammar_pack_mb": 50,
      "dictionary_pack_mb": 85,
      "grammar_paradigm_rows": 1196245,
      "grammar_verbs_only": false
    },
    {
      "code": "fr",
      "language": "French",
      "native_name": "Français",
      "script": "latin",
      "grammar_unimorph": true,
      "dictionary_kaikki": true,
      "examples_tatoeba": true,
      "frequency_list": true,
      "punctuation_model": true,
      "sources_covered": 5,
      "grammar_pack_mb": 13,
      "dictionary_pack_mb": 54,
      "grammar_paradigm_rows": 367732,
      "grammar_verbs_only": true
    },
    {
      "code": "de",
      "language": "German",
      "native_name": "Deutsch",
      "script": "latin",
      "grammar_unimorph": true,
      "dictionary_kaikki": true,
      "examples_tatoeba": true,
      "frequency_list": true,
      "punctuation_model": true,
      "sources_covered": 5,
      "grammar_pack_mb": 19,
      "dictionary_pack_mb": 91,
      "grammar_paradigm_rows": 519143,
      "grammar_verbs_only": false
    },
    {
      "code": "it",
      "language": "Italian",
      "native_name": "Italiano",
      "script": "latin",
      "grammar_unimorph": true,
      "dictionary_kaikki": true,
      "examples_tatoeba": true,
      "frequency_list": true,
      "punctuation_model": true,
      "sources_covered": 5,
      "grammar_pack_mb": 20,
      "dictionary_pack_mb": 71,
      "grammar_paradigm_rows": 509574,
      "grammar_verbs_only": true
    },
    {
      "code": "pt",
      "language": "Portuguese",
      "native_name": "Português",
      "script": "latin",
      "grammar_unimorph": true,
      "dictionary_kaikki": true,
      "examples_tatoeba": true,
      "frequency_list": true,
      "punctuation_model": true,
      "sources_covered": 5,
      "grammar_pack_mb": 11,
      "dictionary_pack_mb": 51,
      "grammar_paradigm_rows": null,
      "grammar_verbs_only": false
    },
    {
      "code": "ru",
      "language": "Russian",
      "native_name": "Русский",
      "script": "cyrillic",
      "grammar_unimorph": true,
      "dictionary_kaikki": true,
      "examples_tatoeba": true,
      "frequency_list": true,
      "punctuation_model": true,
      "sources_covered": 5,
      "grammar_pack_mb": 25,
      "dictionary_pack_mb": 85,
      "grammar_paradigm_rows": null,
      "grammar_verbs_only": false
    },
    {
      "code": "nl",
      "language": "Dutch",
      "native_name": "Nederlands",
      "script": "latin",
      "grammar_unimorph": true,
      "dictionary_kaikki": true,
      "examples_tatoeba": true,
      "frequency_list": false,
      "punctuation_model": true,
      "sources_covered": 4,
      "grammar_pack_mb": 2,
      "dictionary_pack_mb": 28,
      "grammar_paradigm_rows": null,
      "grammar_verbs_only": false
    },
    {
      "code": "pl",
      "language": "Polish",
      "native_name": "Polski",
      "script": "latin",
      "grammar_unimorph": true,
      "dictionary_kaikki": true,
      "examples_tatoeba": true,
      "frequency_list": false,
      "punctuation_model": true,
      "sources_covered": 4,
      "grammar_pack_mb": 7,
      "dictionary_pack_mb": 72,
      "grammar_paradigm_rows": null,
      "grammar_verbs_only": false
    },
    {
      "code": "sv",
      "language": "Swedish",
      "native_name": "Svenska",
      "script": "latin",
      "grammar_unimorph": true,
      "dictionary_kaikki": true,
      "examples_tatoeba": true,
      "frequency_list": false,
      "punctuation_model": false,
      "sources_covered": 3,
      "grammar_pack_mb": 5,
      "dictionary_pack_mb": 31,
      "grammar_paradigm_rows": null,
      "grammar_verbs_only": false
    },
    {
      "code": "ja",
      "language": "Japanese",
      "native_name": "日本語",
      "script": "kana",
      "grammar_unimorph": true,
      "dictionary_kaikki": true,
      "examples_tatoeba": true,
      "frequency_list": false,
      "punctuation_model": true,
      "sources_covered": 4,
      "grammar_pack_mb": 1,
      "dictionary_pack_mb": 45,
      "grammar_paradigm_rows": null,
      "grammar_verbs_only": false
    },
    {
      "code": "zh",
      "language": "Chinese",
      "native_name": "中文",
      "script": "han",
      "grammar_unimorph": false,
      "dictionary_kaikki": true,
      "examples_tatoeba": true,
      "frequency_list": false,
      "punctuation_model": true,
      "sources_covered": 3,
      "grammar_pack_mb": null,
      "dictionary_pack_mb": 145,
      "grammar_paradigm_rows": null,
      "grammar_verbs_only": false
    },
    {
      "code": "ko",
      "language": "Korean",
      "native_name": "한국어",
      "script": "hangul",
      "grammar_unimorph": true,
      "dictionary_kaikki": true,
      "examples_tatoeba": true,
      "frequency_list": false,
      "punctuation_model": true,
      "sources_covered": 4,
      "grammar_pack_mb": 12,
      "dictionary_pack_mb": 19,
      "grammar_paradigm_rows": null,
      "grammar_verbs_only": false
    },
    {
      "code": "ar",
      "language": "Arabic",
      "native_name": "العربية",
      "script": "arabic",
      "grammar_unimorph": true,
      "dictionary_kaikki": true,
      "examples_tatoeba": true,
      "frequency_list": false,
      "punctuation_model": true,
      "sources_covered": 4,
      "grammar_pack_mb": 7,
      "dictionary_pack_mb": 47,
      "grammar_paradigm_rows": null,
      "grammar_verbs_only": false
    },
    {
      "code": "tr",
      "language": "Turkish",
      "native_name": "Türkçe",
      "script": "latin",
      "grammar_unimorph": true,
      "dictionary_kaikki": true,
      "examples_tatoeba": true,
      "frequency_list": false,
      "punctuation_model": true,
      "sources_covered": 4,
      "grammar_pack_mb": 32,
      "dictionary_pack_mb": 32,
      "grammar_paradigm_rows": null,
      "grammar_verbs_only": false
    },
    {
      "code": "sr",
      "language": "Serbian",
      "native_name": "Српски",
      "script": "cyrillic",
      "grammar_unimorph": true,
      "dictionary_kaikki": false,
      "examples_tatoeba": true,
      "frequency_list": false,
      "punctuation_model": true,
      "sources_covered": 3,
      "grammar_pack_mb": 31,
      "dictionary_pack_mb": null,
      "grammar_paradigm_rows": 840799,
      "grammar_verbs_only": false
    },
    {
      "code": "hr",
      "language": "Croatian",
      "native_name": "Hrvatski",
      "script": "latin",
      "grammar_unimorph": true,
      "dictionary_kaikki": false,
      "examples_tatoeba": true,
      "frequency_list": false,
      "punctuation_model": true,
      "sources_covered": 3,
      "grammar_pack_mb": 31,
      "dictionary_pack_mb": null,
      "grammar_paradigm_rows": 840799,
      "grammar_verbs_only": false
    },
    {
      "code": "cs",
      "language": "Czech",
      "native_name": "Čeština",
      "script": "latin",
      "grammar_unimorph": true,
      "dictionary_kaikki": true,
      "examples_tatoeba": true,
      "frequency_list": false,
      "punctuation_model": false,
      "sources_covered": 3,
      "grammar_pack_mb": 5,
      "dictionary_pack_mb": 19,
      "grammar_paradigm_rows": null,
      "grammar_verbs_only": false
    },
    {
      "code": "el",
      "language": "Greek",
      "native_name": "Ελληνικά",
      "script": "greek",
      "grammar_unimorph": true,
      "dictionary_kaikki": true,
      "examples_tatoeba": true,
      "frequency_list": false,
      "punctuation_model": true,
      "sources_covered": 4,
      "grammar_pack_mb": 11,
      "dictionary_pack_mb": 23,
      "grammar_paradigm_rows": null,
      "grammar_verbs_only": false
    },
    {
      "code": "hi",
      "language": "Hindi",
      "native_name": "हिन्दी",
      "script": "devanagari",
      "grammar_unimorph": true,
      "dictionary_kaikki": true,
      "examples_tatoeba": true,
      "frequency_list": false,
      "punctuation_model": true,
      "sources_covered": 4,
      "grammar_pack_mb": 4,
      "dictionary_pack_mb": 16,
      "grammar_paradigm_rows": null,
      "grammar_verbs_only": false
    },
    {
      "code": "vi",
      "language": "Vietnamese",
      "native_name": "Tiếng Việt",
      "script": "latin",
      "grammar_unimorph": false,
      "dictionary_kaikki": false,
      "examples_tatoeba": true,
      "frequency_list": false,
      "punctuation_model": false,
      "sources_covered": 1,
      "grammar_pack_mb": null,
      "dictionary_pack_mb": null,
      "grammar_paradigm_rows": null,
      "grammar_verbs_only": false
    },
    {
      "code": "th",
      "language": "Thai",
      "native_name": "ไทย",
      "script": "thai",
      "grammar_unimorph": false,
      "dictionary_kaikki": false,
      "examples_tatoeba": true,
      "frequency_list": false,
      "punctuation_model": false,
      "sources_covered": 1,
      "grammar_pack_mb": null,
      "dictionary_pack_mb": null,
      "grammar_paradigm_rows": null,
      "grammar_verbs_only": false
    },
    {
      "code": "uk",
      "language": "Ukrainian",
      "native_name": "Українська",
      "script": "cyrillic",
      "grammar_unimorph": true,
      "dictionary_kaikki": true,
      "examples_tatoeba": true,
      "frequency_list": false,
      "punctuation_model": true,
      "sources_covered": 4,
      "grammar_pack_mb": 1,
      "dictionary_pack_mb": null,
      "grammar_paradigm_rows": null,
      "grammar_verbs_only": false
    },
    {
      "code": "ro",
      "language": "Romanian",
      "native_name": "Română",
      "script": "latin",
      "grammar_unimorph": true,
      "dictionary_kaikki": true,
      "examples_tatoeba": true,
      "frequency_list": false,
      "punctuation_model": true,
      "sources_covered": 4,
      "grammar_pack_mb": 3,
      "dictionary_pack_mb": 27,
      "grammar_paradigm_rows": null,
      "grammar_verbs_only": false
    },
    {
      "code": "hu",
      "language": "Hungarian",
      "native_name": "Magyar",
      "script": "latin",
      "grammar_unimorph": true,
      "dictionary_kaikki": true,
      "examples_tatoeba": true,
      "frequency_list": false,
      "punctuation_model": true,
      "sources_covered": 4,
      "grammar_pack_mb": 37,
      "dictionary_pack_mb": null,
      "grammar_paradigm_rows": null,
      "grammar_verbs_only": false
    },
    {
      "code": "fi",
      "language": "Finnish",
      "native_name": "Suomi",
      "script": "latin",
      "grammar_unimorph": false,
      "dictionary_kaikki": true,
      "examples_tatoeba": true,
      "frequency_list": false,
      "punctuation_model": true,
      "sources_covered": 3,
      "grammar_pack_mb": null,
      "dictionary_pack_mb": null,
      "grammar_paradigm_rows": null,
      "grammar_verbs_only": false
    },
    {
      "code": "da",
      "language": "Danish",
      "native_name": "Dansk",
      "script": "latin",
      "grammar_unimorph": true,
      "dictionary_kaikki": true,
      "examples_tatoeba": true,
      "frequency_list": false,
      "punctuation_model": false,
      "sources_covered": 3,
      "grammar_pack_mb": 1,
      "dictionary_pack_mb": null,
      "grammar_paradigm_rows": null,
      "grammar_verbs_only": false
    },
    {
      "code": "no",
      "language": "Norwegian",
      "native_name": "Norsk",
      "script": "latin",
      "grammar_unimorph": true,
      "dictionary_kaikki": true,
      "examples_tatoeba": true,
      "frequency_list": false,
      "punctuation_model": false,
      "sources_covered": 3,
      "grammar_pack_mb": 3,
      "dictionary_pack_mb": null,
      "grammar_paradigm_rows": null,
      "grammar_verbs_only": false
    }
  ]
}