runner/parser: allow on-the-fly grammar constraining

parser/token_parser.go

@@ -33,6 +33,7 @@ type MessageHandler interface {
 
 type ParserInternals interface {
 	AddImplicitStartOrPrefill(prefillString string)
+	ConstraintsAllowed() bool
 }
 
 type ToolParser interface {
@@ -51,6 +52,10 @@ type defaultEngine struct{}
 
 func (defaultEngine) AddImplicitStartOrPrefill(prefillString string) {}
 
+func (defaultEngine) ConstraintsAllowed() bool {
+	return true
+}
+
 type defaultToolParser struct{}
 
 func (defaultToolParser) Add(token string) {}
@@ -104,6 +109,10 @@ func (p *TokenParser) repeatLimitReached(token string) bool {
 	return p.tokenRepeat >= p.repeatLimit
 }
 
+func (p *TokenParser) ConstraintsAllowed() bool {
+	return p.parserEngine.ConstraintsAllowed()
+}
+
 // TODO: update to work with multiple toolcalls - unmarshalling should also happen on parser level
 func (p *TokenParser) Drain() []api.ToolCall {
 	toolName, toolContent := p.toolParser.Drain()