{
  "title": "Selected language token counts for AI Status, October 2026",
  "attribution": "Anri Lombard, tokentax; FLORES+ by Open Language Data Initiative",
  "license": "CC BY-SA 4.0",
  "licenseUrl": "https://creativecommons.org/licenses/by-sa/4.0/",
  "sourceUrl": "https://github.com/Anri-Lombard/tokentax/blob/0efd8729c9e50d69d751976d2473a83d4b157d83/docs/tax.json",
  "checked": "2026-10-18",
  "dataset": "openlanguagedata/flores_plus (devtest, revision e707e62e762a)",
  "adaptation": "Selected twelve languages and four tokenizers. Retained published counts; no tokenizer rerun. Ratios and illustrative pricing computed separately.",
  "limitations": [
    "Exact tiktoken run version not recorded.",
    "Hugging Face tokenizer revisions not pinned.",
    "No per-segment counts in this extract."
  ],
  "tokenizers": {
    "GPT-5 / GPT-4o": [
      "tiktoken",
      "o200k_base"
    ],
    "GPT-4 (2023)": [
      "tiktoken",
      "cl100k_base"
    ],
    "Qwen3.8": [
      "hf",
      "Qwen/Qwen3.8-27B-FP8"
    ],
    "Gemma 4": [
      "hf",
      "google/gemma-4-12b-it"
    ]
  },
  "languages": [
    {
      "code": "eng_Latn",
      "name": "English",
      "script": "Latin",
      "sentences": 1012,
      "tokens": {
        "GPT-5 / GPT-4o": 26873,
        "GPT-4 (2023)": 27182,
        "Qwen3.8": 27530,
        "Gemma 4": 27114
      }
    },
    {
      "code": "spa_Latn",
      "name": "Spanish",
      "script": "Latin",
      "sentences": 1012,
      "tokens": {
        "GPT-5 / GPT-4o": 35379,
        "GPT-4 (2023)": 41939,
        "Qwen3.8": 36388,
        "Gemma 4": 35131
      }
    },
    {
      "code": "deu_Latn",
      "name": "German",
      "script": "Latin",
      "sentences": 1012,
      "tokens": {
        "GPT-5 / GPT-4o": 35224,
        "GPT-4 (2023)": 43103,
        "Qwen3.8": 36164,
        "Gemma 4": 36020
      }
    },
    {
      "code": "rus_Cyrl",
      "name": "Russian",
      "script": "Cyrillic",
      "sentences": 1012,
      "tokens": {
        "GPT-5 / GPT-4o": 38188,
        "GPT-4 (2023)": 66928,
        "Qwen3.8": 37526,
        "Gemma 4": 37023
      }
    },
    {
      "code": "arb_Arab",
      "name": "Standard Arabic",
      "script": "Arabic",
      "sentences": 1012,
      "tokens": {
        "GPT-5 / GPT-4o": 37105,
        "GPT-4 (2023)": 82276,
        "Qwen3.8": 36028,
        "Gemma 4": 39874
      }
    },
    {
      "code": "hin_Deva",
      "name": "Hindi",
      "script": "Devanagari (Nagari)",
      "sentences": 1012,
      "tokens": {
        "GPT-5 / GPT-4o": 42276,
        "GPT-4 (2023)": 129348,
        "Qwen3.8": 55358,
        "Gemma 4": 35521
      }
    },
    {
      "code": "ben_Beng",
      "name": "Bengali",
      "script": "Bengali (Bangla)",
      "sentences": 1012,
      "tokens": {
        "GPT-5 / GPT-4o": 45592,
        "GPT-4 (2023)": 157626,
        "Qwen3.8": 60032,
        "Gemma 4": 32836
      }
    },
    {
      "code": "tam_Taml",
      "name": "Tamil",
      "script": "Tamil",
      "sentences": 1012,
      "tokens": {
        "GPT-5 / GPT-4o": 53199,
        "GPT-4 (2023)": 207772,
        "Qwen3.8": 71218,
        "Gemma 4": 39830
      }
    },
    {
      "code": "tel_Telu",
      "name": "Telugu",
      "script": "Telugu",
      "sentences": 1012,
      "tokens": {
        "GPT-5 / GPT-4o": 51849,
        "GPT-4 (2023)": 225263,
        "Qwen3.8": 78409,
        "Gemma 4": 48148
      }
    },
    {
      "code": "cmn_Hans",
      "name": "Mandarin Chinese",
      "script": "Han (Simplified variant)",
      "sentences": 1012,
      "tokens": {
        "GPT-5 / GPT-4o": 33552,
        "GPT-4 (2023)": 50808,
        "Qwen3.8": 25078,
        "Gemma 4": 29240
      }
    },
    {
      "code": "jpn_Jpan",
      "name": "Japanese",
      "script": "Japanese (alias for Han + Hiragana + Katakana)",
      "sentences": 1012,
      "tokens": {
        "GPT-5 / GPT-4o": 44559,
        "GPT-4 (2023)": 61583,
        "Qwen3.8": 32014,
        "Gemma 4": 32897
      }
    },
    {
      "code": "kor_Hang",
      "name": "Korean",
      "script": "Hangul (Hangŭl, Hangeul)",
      "sentences": 1012,
      "tokens": {
        "GPT-5 / GPT-4o": 39566,
        "GPT-4 (2023)": 64338,
        "Qwen3.8": 33891,
        "Gemma 4": 37268
      }
    }
  ]
}