Token budget filters: count_tokens, truncate_tokens, pack (0359, PR 3)

.github/workflows/benchmark.yml

@@ -132,6 +132,8 @@ jobs:
         # Likewise a base driver that does not know "transformers" (docs/tasks/0360) would render the
         # hf_* chat-template cases in plain Jinja2, or fail on them
         grep -q '"transformers"' ../base/bench/jinja2cpp_bench.cpp || base_cases=../base/bench/cases
+        # And one that does not know "prompt_specials" (docs/tasks/0359) fails on the prompt_budget case
+        grep -q '"prompt_specials"' ../base/bench/jinja2cpp_bench.cpp || base_cases=../base/bench/cases
         python bench/count.py --bench build-base/bench/jinja2cpp_bench --cases-dir "$base_cases" --cache-sim --out base.json
         python bench/count.py --bench build/bench/jinja2cpp_bench --baseline base.json --cache-sim \
           --threshold 0.03 | tee -a "$GITHUB_STEP_SUMMARY"

CMakeLists.txt

@@ -262,6 +262,7 @@ set(JINJA2CPP_SOURCES
     src/out_stream.cpp
     src/prompt_escape.cpp
     src/prompt_filters.cpp
+    src/token_budget.cpp
     src/python_format.cpp
     src/recursion_guard.cpp
     src/render_failure.cpp

bench/README.md

@@ -14,7 +14,12 @@ Each directory in `cases/` is one workload:
 - `settings.json` (optional) sets environment options: `trim_blocks`, `lstrip_blocks`
   and `autoescape`, as in Python's `Environment`, and `"wide": true`, which runs the case
   with `TemplateW` and `std::wstring` data in C++ (Python renders it as usual; the output
-  is compared as UTF-8).
+  is compared as UTF-8). `"cpp_only": true` marks a case that uses C++-only filters
+  (docs/cpp-only-filters.md): Python and the other engines skip it. The prompt filters read
+  `"prompt_specials"` and `"prompt_tags"` (lists of strings, `Settings::promptEscaper`) and
+  `"tokenizer": "words"`, a tokenizer that counts whitespace-separated words; with
+  `"max_token_calls": N`, `--dump-dir` (so `run.py`) fails when one render calls it more than
+  N times.
 
 The cases fall into two groups. The synthetic ones isolate one engine feature each
 (`substitute`, `for_range`, `filters`, `macros`, ...). The realistic ones are what people
@@ -24,6 +29,7 @@ render (docs/tasks/0119):
 |---|---|
 | `chat_llama`, `chat_qwen`, `chat_mistral` | LLM chat templates in the style of Llama 3.1, Qwen 2.5 (ChatML) and Mistral v3, with tool definitions and tool calls (`tojson`, `trim`, slicing, `namespace`, `loop.index0` lookbehind), over a 23-message conversation; `trim_blocks` and `lstrip_blocks` as Hugging Face sets them |
 | `hf_qwen3`, `hf_gpt_oss`, `hf_deepseek_v31`, `hf_mistral_small` | real Hugging Face chat templates (copies from `test/chat_templates/templates/`, Apache-2.0 and MIT model licenses) with the suite's `tools` conversation, in transformers' environment: `"transformers": true` in `settings.json` (`ConfigureForChatTemplates` in C++, the same `ImmutableSandboxedEnvironment` setup in `python_bench.py`; other engines skip them) |
+| `prompt_budget` | 200 retrieved chunks packed into a token budget by score (`pack`), each `neutralize`d and `fence`d; C++ only; the word-counting tokenizer must be called at most once per chunk (docs/tasks/0359) |
 | `html_autoescape` | a shop page with `autoescape` on: escaped user text, `safe`, `e`, imported macros for form fields |
 | `config_file` | an nginx-style config built from many small `include`s and imported macros, `dictsort`, `trim_blocks` |
 | `mitsuhiko_table_wide` | `mitsuhiko_table` with `TemplateW`, so the two compare narrow and wide rendering |

bench/cases/prompt_budget/main.j2

@@ -0,0 +1,5 @@
+{% for d in docs|pack(900, attribute='text', by='score', reverse=true, reserve=60) %}
+<document id="{{ d.id }}">
+{{ d.text|neutralize|fence }}
+</document>
+{% endfor %}

bench/cases/prompt_budget/settings.json

@@ -0,0 +1 @@
+{"cpp_only": true, "tokenizer": "words", "max_token_calls": 200, "prompt_specials": ["<|im_start|>", "<|im_end|>"], "prompt_tags": ["document"]}

bench/jinja2cpp_bench.cpp

@@ -5,7 +5,12 @@
 // so main.j2 can include, import or extend it. An optional settings.json sets environment
 // options (trim_blocks, lstrip_blocks, autoescape, as in Python) and "wide": true, which
 // runs the case with TemplateW and wide strings in the data. bench/python_bench.py runs the
-// same workloads with Python Jinja2, so the two engines can be compared on identical work.
+// same workloads with Python Jinja2, so the two engines can be compared on identical work; a
+// case that uses C++-only filters sets "cpp_only": true and Python skips it. The prompt cases
+// set "prompt_specials" and "prompt_tags" (Settings::promptEscaper) and "tokenizer": "words",
+// a tokenizer that counts whitespace-separated words and counts its calls; --dump-dir fails
+// when a render makes more calls than "max_token_calls" (docs/tasks/0359: pack measures each
+// item at most once).
 //
 // For each case two benchmarks are registered:
 //   Load/<case>    parse main.j2 (Template::Load)
@@ -45,6 +50,7 @@
 #include <jinja2cpp/chat_template.h>
 #endif
 #include <jinja2cpp/filesystem_handler.h>
+#include <jinja2cpp/prompt.h>
 #include <jinja2cpp/string_helpers.h>
 #include <jinja2cpp/template.h>
 #include <jinja2cpp/template_env.h>
@@ -58,6 +64,7 @@
 
 #include <algorithm>
 #include <atomic>
+#include <cstddef>
 #include <chrono>
 #include <cstdint>
 #include <cstdlib>
@@ -263,6 +270,9 @@ struct Case
     jinja2::ValuesMap params;
     nlohmann::ordered_json data; // what params reflect with --data=reflect, in document order
     std::unique_ptr<jinja2::TemplateEnv> env;
+    // The calls of the "words" tokenizer, and the most one render may make (0: no limit)
+    std::shared_ptr<std::atomic<std::size_t>> tokenCalls = std::make_shared<std::atomic<std::size_t>>(0);
+    std::size_t maxTokenCalls = 0;
     // loaded once, for the multi-threaded benchmarks
     std::unique_ptr<jinja2::Template> shared;
     std::unique_ptr<jinja2::TemplateW> sharedW;
@@ -319,6 +329,29 @@ std::unique_ptr<Case> LoadCase(const fs::path& dir, bool reflect)
         envSettings.trimBlocks = settings.value("trim_blocks", false);
         envSettings.lstripBlocks = settings.value("lstrip_blocks", false);
         envSettings.autoescape = settings.value("autoescape", false);
+        if (settings.contains("prompt_specials") || settings.contains("prompt_tags"))
+        {
+            jinja2::PromptEscapeOptions options;
+            options.specialTokens = settings.value("prompt_specials", std::vector<std::string>());
+            options.tags = settings.value("prompt_tags", std::vector<std::string>());
+            envSettings.promptEscaper = jinja2::PromptEscaper(std::move(options));
+        }
+        if (settings.value("tokenizer", std::string()) == "words")
+        {
+            envSettings.tokenizer = jinja2::Tokenizer([calls = result->tokenCalls](std::string_view text) {
+                calls->fetch_add(1, std::memory_order_relaxed);
+                std::size_t words = 0;
+                bool inWord = false;
+                for (const char ch : text)
+                {
+                    const bool space = ch == ' ' || ch == '\n' || ch == '\t';
+                    words += (!space && !inWord) ? 1 : 0;
+                    inWord = !space;
+                }
+                return words;
+            });
+        }
+        result->maxTokenCalls = settings.value("max_token_calls", std::size_t{ 0 });
 #if JINJA2CPP_HAS_CHAT_TEMPLATES
         // A Hugging Face chat template, rendered as transformers does; strftime_now reads the
         // fixed clock python_bench.py sets
@@ -812,12 +845,18 @@ int Dump(const std::vector<std::unique_ptr<Case>>& cases, const fs::path& dumpDi
     {
         std::ofstream out(dumpDir / (c->name + ".txt"), std::ios::binary);
         std::string error;
+        c->tokenCalls->store(0);
         const bool ok = c->wide ? RenderToUtf8<wchar_t>(*c, out, error) : RenderToUtf8<char>(*c, out, error);
         if (!ok)
         {
             std::cerr << c->name << ": " << error << '\n';
             ++failures;
         }
+        else if (c->maxTokenCalls != 0 && c->tokenCalls->load() > c->maxTokenCalls)
+        {
+            std::cerr << c->name << ": " << c->tokenCalls->load() << " tokenizer calls in one render, at most " << c->maxTokenCalls << " allowed\n";
+            ++failures;
+        }
     }
     return failures == 0 ? 0 : 1;
 }

bench/python_bench.py

@@ -23,7 +23,7 @@
 
 
 # settings.json keys that are Environment options; "wide" only switches the C++ driver to TemplateW,
-# "transformers" makes the environment transformers' apply_chat_template one (ConfigureForChatTemplates)
+# "cpp_only" skips the case (is_cpp_only), "transformers" makes the environment transformers' apply_chat_template one (ConfigureForChatTemplates)
 ENV_OPTIONS = ("trim_blocks", "lstrip_blocks", "autoescape")
 
 
@@ -57,6 +57,12 @@ def tojson(x, ensure_ascii=False, indent=None, separators=None, sort_keys=False)
     return env
 
 
+def is_cpp_only(case_dir):
+    """A case that uses C++-only filters (docs/cpp-only-filters.md) has no Python run"""
+    settings_file = case_dir / "settings.json"
+    return bool(settings_file.exists() and json.loads(settings_file.read_text(encoding="utf-8")).get("cpp_only"))
+
+
 def load_case(case_dir):
     templates = {p.name: p.read_text(encoding="utf-8") for p in case_dir.glob("*.j2") if p.name != "main.j2"}
     settings_file = case_dir / "settings.json"
@@ -93,7 +99,7 @@ def main():
     ap.add_argument("--filter", default="")
     args = ap.parse_args()
 
-    cases = sorted(d for d in args.cases_dir.iterdir() if (d / "main.j2").exists())
+    cases = sorted(d for d in args.cases_dir.iterdir() if (d / "main.j2").exists() and not is_cpp_only(d))
     if args.dump_dir:
         args.dump_dir.mkdir(parents=True, exist_ok=True)
         for d in cases:

docs/cpp-only-filters.md

@@ -19,9 +19,7 @@ here (docs/tasks/0233); before, `camelize`, `underscorize`, `escapecpp`, `toxml`
 | `pack(budget, attribute=none, by=none, reverse=false, reserve=0, stop=false)` | The items of a list whose text fits in a token budget, in their original order |
 
 The five prompt filters and the prompt escaping mode are specified at the end of this file
-(docs/tasks/0358, 0359; decided by Ruslan 2026-11-05, docs/api-2.0.md 3.3). `neutralize` and
-`fence` are implemented; until the rest of those tasks land, a template that uses
-`count_tokens`, `truncate_tokens` or `pack` fails to load with `ErrorCode::UnknownFilter`.
+(docs/tasks/0358, 0359; decided by Ruslan 2026-11-05, docs/api-2.0.md 3.3) and implemented.
 
 `toxml` was removed in 2.0: JSON-like data has no single XML mapping, so it would be a format
 of our own to define and support (docs/release-2.0.md, D16). A template that uses it fails

docs/tasks/0358-prompt-autoescape.md

@@ -1,5 +1,6 @@
 ---
-status: in-progress
+status: done
+pr: https://github.com/jinja2cpp/Jinja2Cpp/pull/784
 priority: medium
 area: api
 touches: [src/prompt_escape.cpp, src/prompt_escape.h]

docs/tasks/0408-bench-gate-driver-capabilities.md

@@ -0,0 +1,23 @@
+---
+status: open
+priority: low
+area: ci
+depends: [0359]
+touches: [.github/workflows/benchmark.yml, scripts/preflight.sh, bench/jinja2cpp_bench.cpp]
+shares: [bench/README.md]
+---
+# Detect the base bench driver's features by a capability marker
+
+**Problem.** The instruction gate runs the base (master) driver on the head's `bench/cases`, and
+falls back to the base's own cases when the base driver lacks a feature a new case needs. The
+check is a chain of `grep`s for settings keys (`"transformers"`, `"prompt_specials"`) in the base
+driver's source, duplicated in `.github/workflows/benchmark.yml` and `scripts/preflight.sh`.
+Each new feature case needs a new line in both (found while adding `prompt_budget`, 0359).
+
+**Proposal.** Give the driver a `--capabilities` flag (or a version number) that lists the
+settings keys it understands, and give each case's `settings.json` the keys it needs; the gate
+runs a case on the base only when the base lists them, per case rather than all or nothing.
+Share the logic in one script used by both the workflow and preflight.
+
+**Acceptance.** Adding a case with a new settings key needs no gate edit; existing cases are still
+compared on both sides.

docs/tasks/0409-truncate-tokens-shrink-bound.md

@@ -0,0 +1,21 @@
+---
+status: open
+priority: low
+area: robustness
+depends: [0359]
+touches: [src/token_budget.cpp#TruncateText]
+shares: [test/filters_test.cpp]
+---
+# Bound the host calls of truncate_tokens' shrink loop
+
+**Problem.** When the prefix plus `end` still counts more than `n` tokens (tokens merging across
+the join), `truncate_tokens` lowers the prefix budget and asks the tokenizer again. Each pass
+lowers it by at least one, so a host tokenizer that is not monotonic can make the loop call it
+about `n` times, and `n` comes from the template (0359 verifier, non-blocking).
+
+**Proposal.** Cap the retries (for example at a small constant, then fall back to an empty
+prefix plus `end`, or an error if `end` alone exceeds `n`), or shrink geometrically. Keep the
+`count(result) <= n` guarantee.
+
+**Acceptance.** A test with a non-monotonic fake tokenizer and a large `n` makes a bounded number
+of calls, narrow and wide.

include/jinja2cpp/prompt.h

@@ -3,6 +3,8 @@
 
 #include "config.h"
 
+#include <cstddef>
+#include <functional>
 #include <memory>
 #include <string>
 #include <string_view>
@@ -18,6 +20,8 @@ namespace detail
 {
 class PromptEscaperImpl;
 struct PromptEscaperAccess;
+class TokenizerImpl;
+struct TokenizerAccess;
 } // namespace detail
 
 //! What autoescape escapes for (\ref Settings::autoescapeMode). Jinja2 has Html only
@@ -93,6 +97,57 @@ class JINJA2CPP_EXPORT PromptEscaper
     std::shared_ptr<const detail::PromptEscaperImpl> m_impl;
 };
 
+/*!
+ * \brief The host's token counter for the `count_tokens`, `truncate_tokens` and `pack` filters (docs/cpp-only-filters.md,
+ * "Prompt filters"). Immutable
+ *
+ * A tokenizer is model specific and large, so the library bundles none: the host wraps its own. Copies share the
+ * callbacks and compare equal; tokenizers made apart compare unequal. The default tokenizer has no callback and
+ * estimates `ceil(utf8_bytes / 4)` tokens, which can undercount CJK text, digits and code: a hard context limit needs a
+ * real one.
+ *
+ * ```c++
+ *  env.GetSettings().tokenizer = jinja2::Tokenizer([&tok](std::string_view text) { return tok.Count(text); });
+ * ```
+ *
+ * Callbacks take UTF-8 (a wide template converts its text first) and may be called from renders on several threads at
+ * once: making them thread-safe is the host's job. An exception from a callback fails the render with
+ * ErrorCode::UserCallableException and is kept in ErrorInfo::GetException. Results are not trusted: a prefix length past
+ * the end of the text is clamped, and one inside a UTF-8 sequence is rounded down to its start.
+ */
+class JINJA2CPP_EXPORT Tokenizer
+{
+public:
+    //! The tokens of UTF-8 `text`, without the special tokens a tokenizer adds around a sequence
+    using CountFunction = std::function<std::size_t(std::string_view text)>;
+    //! The bytes of the longest prefix of UTF-8 `text` that is at most `maxTokens` tokens and ends on a token boundary
+    using PrefixFunction = std::function<std::size_t(std::string_view text, std::size_t maxTokens)>;
+
+    //! The byte estimate
+    Tokenizer() = default;
+    /*!
+     * \param count Required. An empty one makes the byte estimate, as the default tokenizer
+     * \param prefix Optional: what `truncate_tokens` cuts with. Without it the filter searches the length with `count`.
+     * Throws std::invalid_argument when given without `count`
+     */
+    explicit Tokenizer(CountFunction count, PrefixFunction prefix = {});
+
+    //! Whether a count function is set
+    explicit operator bool() const noexcept { return static_cast<bool>(m_impl); }
+    //! The tokens of UTF-8 `text`: the estimate when unset, 0 for an empty text without a call. A callback's exception
+    //! is thrown as it is
+    [[nodiscard]] std::size_t Count(std::string_view text) const;
+
+    //! Whether both share their callbacks
+    friend bool operator==(const Tokenizer& lhs, const Tokenizer& rhs) noexcept { return lhs.m_impl == rhs.m_impl; }
+    friend bool operator!=(const Tokenizer& lhs, const Tokenizer& rhs) noexcept { return lhs.m_impl != rhs.m_impl; }
+
+private:
+    friend struct detail::TokenizerAccess;
+
+    std::shared_ptr<const detail::TokenizerImpl> m_impl;
+};
+
 } // namespace v2
 } // namespace jinja2
 

include/jinja2cpp/template_env.h

@@ -153,6 +153,9 @@ struct Settings
     //! The special tokens and tags of the prompts the `neutralize` and `fence` filters protect (docs/cpp-only-filters.md,
     //! "Prompt filters"). None by default, so the filters change nothing until the host registers its model's tokens
     PromptEscaper promptEscaper;
+    //! Counts tokens for the `count_tokens`, `truncate_tokens` and `pack` filters (docs/cpp-only-filters.md). Unset, they
+    //! use the estimate `ceil(utf8_bytes / 4)`
+    Tokenizer tokenizer;
     //! How undefined values behave (Jinja2 `undefined`). A failed use reports ErrorCode::UndefinedError
     UndefinedPolicy undefinedPolicy = UndefinedPolicy::Default;
     //! When `include`, `extends` and `import` look up their template in the environment. `OncePerRender` (default) takes the

scripts/preflight.sh

@@ -141,6 +141,7 @@ if [ "$perf" = 1 ]; then
     # As CI: a base driver that does not know a newer settings.json key counts its own cases
     base_cases=bench/cases
     grep -q '"transformers"' "$wt/bench/jinja2cpp_bench.cpp" || base_cases=$wt/bench/cases
+    grep -q '"prompt_specials"' "$wt/bench/jinja2cpp_bench.cpp" || base_cases=$wt/bench/cases
     if [ $ok = 1 ] \
        && python3 bench/count.py --bench build-preflight-base/bench/jinja2cpp_bench --cases-dir "$base_cases" --out /tmp/preflight-base.json >/dev/null \
        && python3 bench/count.py --bench build-preflight-head/bench/jinja2cpp_bench \

src/filter_factories.cpp

@@ -71,6 +71,7 @@ const std::vector<FilterEntry>& BuiltinFilters()
             Entry<filters::StringConverter, filters::StringConverter::CapitalMode>("capitalize"),
             Entry<filters::StringConverter, filters::StringConverter::CenterMode>("center"),
             Entry<filters::SequenceAccessor, filters::SequenceAccessor::LengthMode>("count"),
+            Entry<filters::TokenBudget, filters::TokenBudget::CountMode>("count_tokens"),
             Entry<filters::Default>("default"),
             Entry<filters::Default>("d"),
             Entry<filters::DictSort>("dictsort"),
@@ -96,6 +97,7 @@ const std::vector<FilterEntry>& BuiltinFilters()
             Entry<filters::SequenceAccessor, filters::SequenceAccessor::MaxItemMode>("max"),
             Entry<filters::SequenceAccessor, filters::SequenceAccessor::MinItemMode>("min"),
             Entry<filters::PromptFilter, filters::PromptFilter::NeutralizeMode>("neutralize"),
+            Entry<filters::TokenBudget, filters::TokenBudget::PackMode>("pack"),
             Entry<filters::PrettyPrint>("pprint"),
             Entry<filters::SequenceAccessor, filters::SequenceAccessor::RandomMode>("random"),
             Entry<filters::Tester, filters::Tester::RejectMode>("reject"),
@@ -116,6 +118,7 @@ const std::vector<FilterEntry>& BuiltinFilters()
             Entry<filters::Serialize, filters::Serialize::YamlMode>("toyaml"),
             Entry<filters::StringConverter, filters::StringConverter::TrimMode>("trim"),
             Entry<filters::StringConverter, filters::StringConverter::TruncateMode>("truncate"),
+            Entry<filters::TokenBudget, filters::TokenBudget::TruncateMode>("truncate_tokens"),
             Entry<filters::SequenceAccessor, filters::SequenceAccessor::UniqueItemsMode>("unique"),
             Entry<filters::StringConverter, filters::StringConverter::UpperMode>("upper"),
             Entry<filters::StringConverter, filters::StringConverter::UrlEncodeMode>("urlencode"),