Sync widgets demo

packages/jinja/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@huggingface/jinja",
 	"packageManager": "[email protected]",
-	"version": "0.2.0",
+	"version": "0.2.1",
 	"description": "A minimalistic JavaScript implementation of the Jinja templating engine, specifically designed for parsing and rendering ML chat templates.",
 	"repository": "https://github.com/huggingface/huggingface.js.git",
 	"publishConfig": {

packages/jinja/src/ast.ts

@@ -120,10 +120,6 @@ export class NumericLiteral extends Literal<number> {
  */
 export class StringLiteral extends Literal<string> {
 	override type = "StringLiteral";
-
-	constructor(value: string) {
-		super(value);
-	}
 }
 
 /**
@@ -133,6 +129,20 @@ export class BooleanLiteral extends Literal<boolean> {
 	override type = "BooleanLiteral";
 }
 
+/**
+ * Represents an array literal in the template.
+ */
+export class ArrayLiteral extends Literal<Expression[]> {
+	override type = "ArrayLiteral";
+}
+
+/**
+ * Represents an object literal in the template.
+ */
+export class ObjectLiteral extends Literal<Map<Expression, Expression>> {
+	override type = "ObjectLiteral";
+}
+
 /**
  * An operation with two sides, separated by an operator.
  * Note: Either side can be a Complex Expression, with order
@@ -159,7 +169,7 @@ export class FilterExpression extends Expression {
 
 	constructor(
 		public operand: Expression,
-		public filter: Identifier // TODO: Add support for non-identifier filters
+		public filter: Identifier | CallExpression
 	) {
 		super();
 	}

packages/jinja/src/lexer.ts

@@ -17,6 +17,8 @@ export const TOKEN_TYPES = Object.freeze({
 	CloseExpression: "CloseExpression", // }}
 	OpenSquareBracket: "OpenSquareBracket", // [
 	CloseSquareBracket: "CloseSquareBracket", // ]
+	OpenCurlyBracket: "OpenCurlyBracket", // {
+	CloseCurlyBracket: "CloseCurlyBracket", // }
 	Comma: "Comma", // ,
 	Dot: "Dot", // .
 	Colon: "Colon", // :
@@ -104,6 +106,8 @@ const ORDERED_MAPPING_TABLE: [string, TokenType][] = [
 	// Single character tokens
 	["(", TOKEN_TYPES.OpenParen],
 	[")", TOKEN_TYPES.CloseParen],
+	["{", TOKEN_TYPES.OpenCurlyBracket],
+	["}", TOKEN_TYPES.CloseCurlyBracket],
 	["[", TOKEN_TYPES.OpenSquareBracket],
 	["]", TOKEN_TYPES.CloseSquareBracket],
 	[",", TOKEN_TYPES.Comma],
@@ -154,19 +158,25 @@ function preprocess(template: string, options: PreprocessOptions = {}): string {
 		template = template.slice(0, -1);
 	}
 
-	if (options.trim_blocks) {
-		// If an application configures Jinja to trim_blocks, the first newline after
-		// a template tag is removed automatically (like in PHP).
-		template = template.replace(/%}\n/g, "%}");
-	}
+	// Replace all comments with a placeholder
+	// This ensures that comments don't interfere with the following options
+	template = template.replace(/{#.*?#}/gs, "{##}");
+
 	if (options.lstrip_blocks) {
 		// The lstrip_blocks option can also be set to strip tabs and spaces from the
 		// beginning of a line to the start of a block. (Nothing will be stripped if
 		// there are other characters before the start of the block.)
-		template = template.replace(/^[ \t]*{%/gm, "{%");
+		template = template.replace(/^[ \t]*({[#%])/gm, "$1");
+	}
+
+	if (options.trim_blocks) {
+		// If an application configures Jinja to trim_blocks, the first newline after
+		// a template tag is removed automatically (like in PHP).
+		template = template.replace(/([#%]})\n/g, "$1");
 	}
 
 	return template
+		.replace(/{##}/g, "") // Remove comments
 		.replace(/-%}\s*/g, "%}")
 		.replace(/\s*{%-/g, "{%")
 		.replace(/-}}\s*/g, "}}")
@@ -283,9 +293,9 @@ export function tokenize(source: string, options: PreprocessOptions = {}): Token
 			}
 		}
 
-		if (char === "'") {
+		if (char === "'" || char === '"') {
 			++cursorPosition; // Skip the opening quote
-			const str = consumeWhile((char) => char !== "'");
+			const str = consumeWhile((c) => c !== char);
 			tokens.push(new Token(str, TOKEN_TYPES.StringLiteral));
 			++cursorPosition; // Skip the closing quote
 			continue;

packages/jinja/src/parser.ts

@@ -12,6 +12,8 @@ import {
 	NumericLiteral,
 	StringLiteral,
 	BooleanLiteral,
+	ArrayLiteral,
+	ObjectLiteral,
 	BinaryExpression,
 	FilterExpression,
 	TestExpression,
@@ -197,7 +199,16 @@ export function parse(tokens: Token[]): Program {
 
 	function parseExpression(): Statement {
 		// Choose parse function with lowest precedence
-		return parseLogicalOrExpression();
+		const a = parseLogicalOrExpression();
+		if (is(TOKEN_TYPES.If)) {
+			// Ternary expression
+			++current; // consume if
+			const predicate = parseLogicalOrExpression();
+			expect(TOKEN_TYPES.Else, "Expected else token");
+			const b = parseLogicalOrExpression();
+			return new If(predicate, [a], [b]);
+		}
+		return a;
 	}
 
 	function parseLogicalOrExpression(): Statement {
@@ -423,11 +434,14 @@ export function parse(tokens: Token[]): Program {
 		while (is(TOKEN_TYPES.Pipe)) {
 			// Support chaining filters
 			++current; // consume pipe
-			const filter = parsePrimaryExpression(); // should be an identifier
+			let filter = parsePrimaryExpression(); // should be an identifier
 			if (!(filter instanceof Identifier)) {
 				throw new SyntaxError(`Expected identifier for the filter`);
 			}
-			operand = new FilterExpression(operand, filter);
+			if (is(TOKEN_TYPES.OpenParen)) {
+				filter = parseCallExpression(filter);
+			}
+			operand = new FilterExpression(operand, filter as Identifier | CallExpression);
 		}
 		return operand;
 	}
@@ -457,6 +471,39 @@ export function parse(tokens: Token[]): Program {
 				++current; // consume closing parenthesis
 				return expression;
 			}
+			case TOKEN_TYPES.OpenSquareBracket: {
+				++current; // consume opening square bracket
+
+				const values = [];
+				while (!is(TOKEN_TYPES.CloseSquareBracket)) {
+					values.push(parseExpression());
+
+					if (is(TOKEN_TYPES.Comma)) {
+						++current; // consume comma
+					}
+				}
+				++current; // consume closing square bracket
+
+				return new ArrayLiteral(values);
+			}
+			case TOKEN_TYPES.OpenCurlyBracket: {
+				++current; // consume opening curly bracket
+
+				const values = new Map();
+				while (!is(TOKEN_TYPES.CloseCurlyBracket)) {
+					const key = parseExpression();
+					expect(TOKEN_TYPES.Colon, "Expected colon between key and value in object literal");
+					const value = parseExpression();
+					values.set(key, value);
+
+					if (is(TOKEN_TYPES.Comma)) {
+						++current; // consume comma
+					}
+				}
+				++current; // consume closing curly bracket
+
+				return new ObjectLiteral(values);
+			}
 			default:
 				throw new SyntaxError(`Unexpected token: ${token.type}`);
 		}

packages/jinja/src/runtime.ts

@@ -2,6 +2,7 @@ import type {
 	NumericLiteral,
 	StringLiteral,
 	BooleanLiteral,
+	ArrayLiteral,
 	Statement,
 	Program,
 	If,
@@ -16,6 +17,7 @@ import type {
 	UnaryExpression,
 	SliceExpression,
 	KeywordArgumentExpression,
+	ObjectLiteral,
 } from "./ast";
 import { slice, titleCase } from "./utils";
 
@@ -125,6 +127,18 @@ export class ObjectValue extends RuntimeValue<Map<string, AnyRuntimeValue>> {
 	override __bool__(): BooleanValue {
 		return new BooleanValue(this.value.size > 0);
 	}
+
+	override builtins: Map<string, AnyRuntimeValue> = new Map<string, AnyRuntimeValue>([
+		[
+			"get",
+			new FunctionValue(([key, defaultValue]) => {
+				if (!(key instanceof StringValue)) {
+					throw new Error(`Object key must be a string: got ${key.type}`);
+				}
+				return this.value.get(key.value) ?? defaultValue ?? new NullValue();
+			}),
+		],
+	]);
 }
 
 /**
@@ -190,13 +204,61 @@ export class Environment {
 		],
 	]);
 
+	/**
+	 * The tests available in this environment.
+	 */
+	tests: Map<string, (...value: AnyRuntimeValue[]) => boolean> = new Map([
+		["boolean", (operand) => operand.type === "BooleanValue"],
+		["callable", (operand) => operand instanceof FunctionValue],
+		[
+			"odd",
+			(operand) => {
+				if (operand.type !== "NumericValue") {
+					throw new Error(`Cannot apply test "odd" to type: ${operand.type}`);
+				}
+				return (operand as NumericValue).value % 2 !== 0;
+			},
+		],
+		[
+			"even",
+			(operand) => {
+				if (operand.type !== "NumericValue") {
+					throw new Error(`Cannot apply test "even" to type: ${operand.type}`);
+				}
+				return (operand as NumericValue).value % 2 === 0;
+			},
+		],
+		["false", (operand) => operand.type === "BooleanValue" && !(operand as BooleanValue).value],
+		["true", (operand) => operand.type === "BooleanValue" && (operand as BooleanValue).value],
+		["number", (operand) => operand.type === "NumericValue"],
+		["integer", (operand) => operand.type === "NumericValue" && Number.isInteger((operand as NumericValue).value)],
+		["iterable", (operand) => operand instanceof ArrayValue || operand instanceof StringValue],
+		[
+			"lower",
+			(operand) => {
+				const str = (operand as StringValue).value;
+				return operand.type === "StringValue" && str === str.toLowerCase();
+			},
+		],
+		[
+			"upper",
+			(operand) => {
+				const str = (operand as StringValue).value;
+				return operand.type === "StringValue" && str === str.toUpperCase();
+			},
+		],
+		["none", (operand) => operand.type === "NullValue"],
+		["defined", (operand) => operand.type !== "UndefinedValue"],
+		["undefined", (operand) => operand.type === "UndefinedValue"],
+		["equalto", (a, b) => a.value === b.value],
+	]);
+
 	constructor(public parent?: Environment) {}
 
 	/**
 	 * Set the value of a variable in the current environment.
 	 */
-	// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/explicit-module-boundary-types
-	set(name: string, value: any): AnyRuntimeValue {
+	set(name: string, value: unknown): AnyRuntimeValue {
 		return this.declareVariable(name, convertToRuntimeValues(value));
 	}
 
@@ -215,16 +277,11 @@ export class Environment {
 	// }
 
 	/**
-	 * Declare if doesn't exist, assign otherwise.
+	 * Set variable in the current scope.
+	 * See https://jinja.palletsprojects.com/en/3.0.x/templates/#assignments for more information.
 	 */
 	setVariable(name: string, value: AnyRuntimeValue): AnyRuntimeValue {
-		let env: Environment | undefined;
-		try {
-			env = this.resolve(name);
-		} catch {
-			/* empty */
-		}
-		(env ?? this).variables.set(name, value);
+		this.variables.set(name, value);
 		return value;
 	}
 
@@ -247,7 +304,11 @@ export class Environment {
 	}
 
 	lookupVariable(name: string): AnyRuntimeValue {
-		return this.resolve(name).variables.get(name) ?? new NullValue();
+		try {
+			return this.resolve(name).variables.get(name) ?? new UndefinedValue();
+		} catch {
+			return new UndefinedValue();
+		}
 	}
 }
 
@@ -318,6 +379,12 @@ export class Interpreter {
 				case "<=":
 					return new BooleanValue(left.value <= right.value);
 			}
+		} else if (left instanceof ArrayValue && right instanceof ArrayValue) {
+			// Evaluate array operands with binary operator.
+			switch (node.operator.value) {
+				case "+":
+					return new ArrayValue(left.value.concat(right.value));
+			}
 		} else if (right instanceof ArrayValue) {
 			const member = right.value.find((x) => x.value === left.value) !== undefined;
 			switch (node.operator.value) {
@@ -345,6 +412,15 @@ export class Interpreter {
 			}
 		}
 
+		if (left instanceof StringValue && right instanceof ObjectValue) {
+			switch (node.operator.value) {
+				case "in":
+					return new BooleanValue(right.value.has(left.value));
+				case "not in":
+					return new BooleanValue(!right.value.has(left.value));
+			}
+		}
+
 		throw new SyntaxError(`Unknown operator "${node.operator.value}" between ${left.type} and ${right.type}`);
 	}
 
@@ -365,62 +441,118 @@ export class Interpreter {
 		//   return filter.value([operand], environment);
 
 		// https://jinja.palletsprojects.com/en/3.0.x/templates/#list-of-builtin-filters
-		if (operand instanceof ArrayValue) {
-			switch (node.filter.value) {
-				case "first":
-					return operand.value[0];
-				case "last":
-					return operand.value[operand.value.length - 1];
-				case "length":
-					return new NumericValue(operand.value.length);
-				case "reverse":
-					return new ArrayValue(operand.value.reverse());
-				case "sort":
-					return new ArrayValue(
-						operand.value.sort((a, b) => {
-							if (a.type !== b.type) {
-								throw new Error(`Cannot compare different types: ${a.type} and ${b.type}`);
-							}
-							switch (a.type) {
-								case "NumericValue":
-									return (a as NumericValue).value - (b as NumericValue).value;
-								case "StringValue":
-									return (a as StringValue).value.localeCompare((b as StringValue).value);
-								default:
-									throw new Error(`Cannot compare type: ${a.type}`);
-							}
-						})
-					);
-				default:
-					throw new Error(`Unknown ArrayValue filter: ${node.filter.value}`);
+
+		if (node.filter.type === "Identifier") {
+			const filter = node.filter as Identifier;
+
+			if (operand instanceof ArrayValue) {
+				switch (filter.value) {
+					case "list":
+						return operand;
+					case "first":
+						return operand.value[0];
+					case "last":
+						return operand.value[operand.value.length - 1];
+					case "length":
+						return new NumericValue(operand.value.length);
+					case "reverse":
+						return new ArrayValue(operand.value.reverse());
+					case "sort":
+						return new ArrayValue(
+							operand.value.sort((a, b) => {
+								if (a.type !== b.type) {
+									throw new Error(`Cannot compare different types: ${a.type} and ${b.type}`);
+								}
+								switch (a.type) {
+									case "NumericValue":
+										return (a as NumericValue).value - (b as NumericValue).value;
+									case "StringValue":
+										return (a as StringValue).value.localeCompare((b as StringValue).value);
+									default:
+										throw new Error(`Cannot compare type: ${a.type}`);
+								}
+							})
+						);
+					default:
+						throw new Error(`Unknown ArrayValue filter: ${filter.value}`);
+				}
+			} else if (operand instanceof StringValue) {
+				switch (filter.value) {
+					case "length":
+						return new NumericValue(operand.value.length);
+					case "upper":
+						return new StringValue(operand.value.toUpperCase());
+					case "lower":
+						return new StringValue(operand.value.toLowerCase());
+					case "title":
+						return new StringValue(titleCase(operand.value));
+					case "capitalize":
+						return new StringValue(operand.value.charAt(0).toUpperCase() + operand.value.slice(1));
+					case "trim":
+						return new StringValue(operand.value.trim());
+					default:
+						throw new Error(`Unknown StringValue filter: ${filter.value}`);
+				}
+			} else if (operand instanceof NumericValue) {
+				switch (filter.value) {
+					case "abs":
+						return new NumericValue(Math.abs(operand.value));
+					default:
+						throw new Error(`Unknown NumericValue filter: ${filter.value}`);
+				}
 			}
-		} else if (operand instanceof StringValue) {
-			switch (node.filter.value) {
-				case "length":
-					return new NumericValue(operand.value.length);
-				case "upper":
-					return new StringValue(operand.value.toUpperCase());
-				case "lower":
-					return new StringValue(operand.value.toLowerCase());
-				case "title":
-					return new StringValue(titleCase(operand.value));
-				case "capitalize":
-					return new StringValue(operand.value.charAt(0).toUpperCase() + operand.value.slice(1));
-				case "trim":
-					return new StringValue(operand.value.trim());
-				default:
-					throw new Error(`Unknown StringValue filter: ${node.filter.value}`);
+			throw new Error(`Cannot apply filter "${filter.value}" to type: ${operand.type}`);
+		} else if (node.filter.type === "CallExpression") {
+			const filter = node.filter as CallExpression;
+
+			if (filter.callee.type !== "Identifier") {
+				throw new Error(`Unknown filter: ${filter.callee.type}`);
 			}
-		} else if (operand instanceof NumericValue) {
-			switch (node.filter.value) {
-				case "abs":
-					return new NumericValue(Math.abs(operand.value));
-				default:
-					throw new Error(`Unknown NumericValue filter: ${node.filter.value}`);
+			const filterName = (filter.callee as Identifier).value;
+
+			if (operand instanceof ArrayValue) {
+				switch (filterName) {
+					case "selectattr": {
+						if (operand.value.some((x) => !(x instanceof ObjectValue))) {
+							throw new Error("`selectattr` can only be applied to array of objects");
+						}
+						if (filter.args.some((x) => x.type !== "StringLiteral")) {
+							throw new Error("arguments of `selectattr` must be strings");
+						}
+
+						const [attr, testName, value] = filter.args.map((x) => this.evaluate(x, environment)) as StringValue[];
+
+						let testFunction: (...x: AnyRuntimeValue[]) => boolean;
+						if (testName) {
+							// Get the test function from the environment
+							const test = environment.tests.get(testName.value);
+							if (!test) {
+								throw new Error(`Unknown test: ${testName.value}`);
+							}
+							testFunction = test;
+						} else {
+							// Default to truthiness of first argument
+							testFunction = (...x: AnyRuntimeValue[]) => x[0].__bool__().value;
+						}
+
+						// Filter the array using the test function
+						const filtered = (operand.value as ObjectValue[]).filter((item) => {
+							const a = item.value.get(attr.value);
+							if (a) {
+								return testFunction(a, value);
+							}
+							return false;
+						});
+
+						return new ArrayValue(filtered);
+					}
+				}
+				throw new Error(`Unknown ArrayValue filter: ${filterName}`);
+			} else {
+				throw new Error(`Cannot apply filter "${filterName}" to type: ${operand.type}`);
 			}
 		}
-
-		throw new Error(`Cannot apply filter "${node.filter.value}" to type: ${operand.type}`);
+		throw new Error(`Unknown filter: ${node.filter.type}`);
 	}
 
 	/**
@@ -431,65 +563,13 @@ export class Interpreter {
 		// https://jinja.palletsprojects.com/en/3.0.x/templates/#list-of-builtin-tests
 		//
 		// TODO: Add support for non-identifier tests. e.g., divisibleby(number)
+		const operand = this.evaluate(node.operand, environment);
 
-		const result: boolean = (() => {
-			try {
-				const operand = this.evaluate(node.operand, environment);
-
-				switch (node.test.value) {
-					case "boolean":
-						return operand.type === "BooleanValue";
-					case "callable":
-						return operand instanceof FunctionValue;
-					case "odd":
-						if (operand.type !== "NumericValue") {
-							throw new Error(`Cannot apply test "odd" to type: ${operand.type}`);
-						}
-						return (operand as NumericValue).value % 2 !== 0;
-					case "even":
-						if (operand.type !== "NumericValue") {
-							throw new Error(`Cannot apply test "even" to type: ${operand.type}`);
-						}
-						return (operand as NumericValue).value % 2 === 0;
-					case "false":
-						return operand.type === "BooleanValue" && !(operand as BooleanValue).value;
-					case "true":
-						return operand.type === "BooleanValue" && (operand as BooleanValue).value;
-					case "number":
-						return operand.type === "NumericValue";
-					case "integer":
-						return operand.type === "NumericValue" && Number.isInteger((operand as NumericValue).value);
-					case "iterable":
-						return operand instanceof ArrayValue || operand instanceof StringValue;
-					case "lower": {
-						const str = (operand as StringValue).value;
-						return operand.type === "StringValue" && str === str.toLowerCase();
-					}
-					case "upper": {
-						const str = (operand as StringValue).value;
-						return operand.type === "StringValue" && str === str.toUpperCase();
-					}
-					case "none":
-						return operand.type === "NullValue";
-					case "defined":
-						return true;
-					case "undefined":
-						return false;
-				}
-				throw new Error(`Unknown test: ${node.test.value}`);
-			} catch (e) {
-				if (node.operand.type === "Identifier") {
-					// Special cases where we want to check if a variable is defined
-					if (node.test.value === "defined") {
-						return false;
-					} else if (node.test.value === "undefined") {
-						return true;
-					}
-				}
-				throw e;
-			}
-		})();
-
+		const test = environment.tests.get(node.test.value);
+		if (!test) {
+			throw new Error(`Unknown test: ${node.test.value}`);
+		}
+		const result = test(operand);
 		return new BooleanValue(node.negate ? !result : result);
 	}
 
@@ -519,7 +599,7 @@ export class Interpreter {
 		for (const statement of statements) {
 			const lastEvaluated = this.evaluate(statement, environment);
 
-			if (lastEvaluated.type !== "NullValue") {
+			if (lastEvaluated.type !== "NullValue" && lastEvaluated.type !== "UndefinedValue") {
 				result += lastEvaluated.value;
 			}
 		}
@@ -623,10 +703,8 @@ export class Interpreter {
 			}
 			value = object.builtins.get(property.value);
 		}
-		if (!(value instanceof RuntimeValue)) {
-			throw new Error(`${object.type} has no property '${property.value}'`);
-		}
-		return value;
+
+		return value instanceof RuntimeValue ? value : new UndefinedValue();
 	}
 
 	private evaluateSet(node: SetStatement, environment: Environment): NullValue {
@@ -719,6 +797,19 @@ export class Interpreter {
 				return new StringValue((statement as StringLiteral).value);
 			case "BooleanLiteral":
 				return new BooleanValue((statement as BooleanLiteral).value);
+			case "ArrayLiteral":
+				return new ArrayValue((statement as ArrayLiteral).value.map((x) => this.evaluate(x, environment)));
+			case "ObjectLiteral": {
+				const mapping = new Map();
+				for (const [key, value] of (statement as ObjectLiteral).value) {
+					const evaluatedKey = this.evaluate(key, environment);
+					if (!(evaluatedKey instanceof StringValue)) {
+						throw new Error(`Object keys must be strings: got ${evaluatedKey.type}`);
+					}
+					mapping.set(evaluatedKey.value, this.evaluate(value, environment));
+				}
+				return new ObjectValue(mapping);
+			}
 			case "Identifier":
 				return this.evaluateIdentifier(statement as Identifier, environment);
 			case "CallExpression":

packages/jinja/test/e2e.test.js

@@ -18,6 +18,64 @@ const EXAMPLE_CHAT_WITH_SYSTEM = [
 	...EXAMPLE_CHAT,
 ];
 
+const EXAMPLE_FUNCTION_CALLING = [
+	{
+		role: "assistant",
+		content: null,
+		tool_calls: [
+			{
+				type: "function",
+				function: {
+					name: "get_current_weather",
+					arguments: '{\n  "location": "Hanoi"\n}',
+				},
+			},
+		],
+	},
+	{ role: "user", content: "what's the weather like in Hanoi?" },
+];
+
+// Example adapted from https://huggingface.co/fireworks-ai/firefunction-v1
+const EXAMPLE_FUNCTION_SPEC = [
+	{
+		name: "get_stock_price",
+		description: "Get the current stock price",
+		parameters: {
+			type: "object",
+			properties: {
+				symbol: {
+					type: "string",
+					description: "The stock symbol, e.g. AAPL, GOOG",
+				},
+			},
+			required: ["symbol"],
+		},
+	},
+	{
+		name: "check_word_anagram",
+		description: "Check if two words are anagrams of each other",
+		parameters: {
+			type: "object",
+			properties: {
+				word1: {
+					type: "string",
+					description: "The first word",
+				},
+				word2: {
+					type: "string",
+					description: "The second word",
+				},
+			},
+			required: ["word1", "word2"],
+		},
+	},
+];
+const EXAMPLE_FUNCTION_CALLING_WITH_SYSTEM = [
+	{ role: "functions", content: JSON.stringify(EXAMPLE_FUNCTION_SPEC, null, 4) },
+	{ role: "system", content: "You are a helpful assistant with access to functions. Use them if required." },
+	{ role: "user", content: "Hi, can you tell me the current stock price of AAPL?" },
+];
+
 /**
  * Defined in https://github.com/huggingface/transformers
  * Keys correspond to `model_type` in the transformers repo.
@@ -286,6 +344,38 @@ const TEST_CUSTOM_TEMPLATES = Object.freeze({
 		},
 		target: `<｜begin▁of▁sentence｜>You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\n### Instruction:\nHello, how are you?\n### Response:\nI'm doing great. How can I help you today?\n<|EOT|>\n### Instruction:\nI'd like to show off how chat templating works!\n`,
 	},
+	"meetkai/functionary-medium-v2.2": {
+		chat_template: `{#v2.2#}\n{% for message in messages %}\n{% if message['role'] == 'user' or message['role'] == 'system' %}\n{{ '<|from|>' + message['role'] + '\n<|recipient|>all\n<|content|>' + message['content'] + '\n' }}{% elif message['role'] == 'tool' %}\n{{ '<|from|>' + message['name'] + '\n<|recipient|>all\n<|content|>' + message['content'] + '\n' }}{% else %}\n{% set contain_content='no'%}\n{% if message['content'] is not none %}\n{{ '<|from|>assistant\n<|recipient|>all\n<|content|>' + message['content'] }}{% set contain_content='yes'%}\n{% endif %}\n{% if 'tool_calls' in message and message['tool_calls'] is not none %}\n{% for tool_call in message['tool_calls'] %}\n{% set prompt='<|from|>assistant\n<|recipient|>' + tool_call['function']['name'] + '\n<|content|>' + tool_call['function']['arguments'] %}\n{% if loop.index == 1 and contain_content == "no" %}\n{{ prompt }}{% else %}\n{{ '\n' + prompt}}{% endif %}\n{% endfor %}\n{% endif %}\n{{ '<|stop|>\n' }}{% endif %}\n{% endfor %}\n{% if add_generation_prompt %}{{ '<|from|>assistant\n<|recipient|>' }}{% endif %}`,
+		data: {
+			messages: EXAMPLE_FUNCTION_CALLING,
+			bos_token: "<s>",
+			eos_token: "</s>",
+			add_generation_prompt: false,
+		},
+		target: `<|from|>assistant\n<|recipient|>get_current_weather\n<|content|>{\n  "location": "Hanoi"\n}<|stop|>\n<|from|>user\n<|recipient|>all\n<|content|>what's the weather like in Hanoi?\n`,
+	},
+	"fireworks-ai/firefunction-v1": {
+		chat_template: `{%- set message_roles = ['SYSTEM', 'FUNCTIONS', 'USER', 'ASSISTANT', 'TOOL'] -%}\n{%- set ns = namespace(seen_non_system=false, messages=messages, content='', functions=[]) -%}\n{{ bos_token }}\n{#- Basic consistency checks -#}\n{%- if not ns.messages -%}\n  {{ raise_exception('No messages') }}\n{%- endif -%}\n{%- if ns.messages[0]['role'] | upper != 'SYSTEM' -%}\n  {%- set ns.messages = [{'role': 'SYSTEM', 'content': 'You are a helpful assistant with access to functions. Use them if required.'}] + ns.messages -%}\n{%- endif -%}\n{%- if ns.messages | length < 2 or ns.messages[0]['role'] | upper != 'SYSTEM' or ns.messages[1]['role'] | upper != 'FUNCTIONS' -%}\n  {{ raise_exception('Expected either "functions" or ["system", "functions"] as the first messages') }}\n{%- endif -%}\n{%- for message in ns.messages -%}\n  {%- set role = message['role'] | upper -%}\n  {#- Validation -#}\n  {%- if role not in message_roles -%}\n    {{ raise_exception('Invalid role ' + message['role'] + '. Only ' + message_roles + ' are supported.') }}\n  {%- endif -%}\n  {%- set ns.content = message['content'] if message.get('content') else '' -%}\n  {#- Move tool calls inside the content -#}\n  {%- if 'tool_calls' in message -%}\n    {%- for call in message['tool_calls'] -%}\n      {%- set ns.content = ns.content + '<functioncall>{"name": "' + call['function']['name'] + '", "arguments": ' + call['function']['arguments'] + '}' -%}\n    {%- endfor -%}\n  {%- endif -%}\n  {%- if role == 'ASSISTANT' and '<functioncall>' not in ns.content -%}\n    {%- set ns.content = '<plain>' + ns.content -%}\n  {%- endif -%}\n  {%- if role == 'ASSISTANT' -%}\n    {%- set ns.content = ns.content + eos_token -%}\n  {%- endif -%}\n  {{ role }}: {{ ns.content }}{{ '\\n\\n' }}\n{%- endfor -%}\nASSISTANT:{{ ' ' }}\n`,
+		data: {
+			messages: EXAMPLE_FUNCTION_CALLING_WITH_SYSTEM,
+			bos_token: "<s>",
+			eos_token: "</s>",
+			add_generation_prompt: false,
+		},
+		target: `<s>SYSTEM: You are a helpful assistant with access to functions. Use them if required.\n\nFUNCTIONS: [\n    {\n        "name": "get_stock_price",\n        "description": "Get the current stock price",\n        "parameters": {\n            "type": "object",\n            "properties": {\n                "symbol": {\n                    "type": "string",\n                    "description": "The stock symbol, e.g. AAPL, GOOG"\n                }\n            },\n            "required": [\n                "symbol"\n            ]\n        }\n    },\n    {\n        "name": "check_word_anagram",\n        "description": "Check if two words are anagrams of each other",\n        "parameters": {\n            "type": "object",\n            "properties": {\n                "word1": {\n                    "type": "string",\n                    "description": "The first word"\n                },\n                "word2": {\n                    "type": "string",\n                    "description": "The second word"\n                }\n            },\n            "required": [\n                "word1",\n                "word2"\n            ]\n        }\n    }\n]\n\nSYSTEM: You are a helpful assistant with access to functions. Use them if required.\n\nUSER: Hi, can you tell me the current stock price of AAPL?\n\nASSISTANT: `,
+	},
+	"maywell/PiVoT-MoE": {
+		chat_template: `{{ (messages|selectattr('role', 'equalto', 'system')|list|last).content|trim if (messages|selectattr('role', 'equalto', 'system')|list) else '' }}{% for message in messages %}{% if message['role'] == 'system' %}{{ message['content']|trim }}{% elif message['role'] == 'user' %}### Instruction: {{ message['content']|trim }}{% elif message['role'] == 'assistant' %}### Response: {{ message['content']|trim }}{% elif message['role'] == 'user_context' %}### Input: {{ message['content']|trim }}{% endif %}{% if not loop.last %}\n{% endif %}{% endfor %}{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}### Response:{% endif %}`,
+		data: {
+			messages: EXAMPLE_CHAT_WITH_SYSTEM,
+			bos_token: "<s>",
+			eos_token: "</s>",
+			add_generation_prompt: false,
+		},
+		// NOTE: There is a bug in the model's chat template which causes the system prompt
+		// to be repeated twice. We replicate this behaviour here.
+		target: `You are a friendly chatbot who always responds in the style of a pirateYou are a friendly chatbot who always responds in the style of a pirate### Instruction: Hello, how are you?### Response: I'm doing great. How can I help you today?### Instruction: I'd like to show off how chat templating works!`,
+	},
 });
 
 describe("End-to-end tests", () => {

packages/jinja/test/interpreter.test.js

@@ -12,6 +12,7 @@ describe("Test interpreter options", () => {
 		const EXAMPLE_FOR_TEMPLATE_2 = `{% for item in seq -%}\n    {{ item }}\n{% endfor %}`;
 		const EXAMPLE_FOR_TEMPLATE_3 = `{% for item in seq %}\n    {{ item }}\n{%- endfor %}`;
 		const EXAMPLE_FOR_TEMPLATE_4 = `{% for item in seq -%}\n    {{ item }}\n{%- endfor %}`;
+		const EXAMPLE_COMMENT_TEMPLATE = `    {# comment #}\n  {# {% if true %} {% endif %} #}\n`;
 
 		const seq = [1, 2, 3, 4, 5, 6, 7, 8, 9];
 
@@ -98,6 +99,32 @@ describe("Test interpreter options", () => {
 				options: {},
 				target: `123456789`,
 			},
+
+			// Comment tests
+			{
+				template: EXAMPLE_COMMENT_TEMPLATE,
+				data: {},
+				options: {},
+				target: `    \n  `,
+			},
+			{
+				template: EXAMPLE_COMMENT_TEMPLATE,
+				data: {},
+				options: { lstrip_blocks: true },
+				target: `\n`,
+			},
+			{
+				template: EXAMPLE_COMMENT_TEMPLATE,
+				data: {},
+				options: { trim_blocks: true },
+				target: `      `,
+			},
+			{
+				template: EXAMPLE_COMMENT_TEMPLATE,
+				data: {},
+				options: { lstrip_blocks: true, trim_blocks: true },
+				target: ``,
+			},
 		];
 
 		for (const test of TESTS) {

packages/jinja/test/templates.test.js

@@ -39,6 +39,8 @@ const TEST_STRINGS = {
 
 	// Strings
 	STRINGS: `{{ 'Bye' }}{{ bos_token + '[INST] ' }}`,
+	STRINGS_1: `|{{ "test" }}|{{ "a" + 'b' + "c" }}|{{ '"' + "'" }}|{{ '\\'' }}|{{ "\\"" }}|`,
+	STRINGS_2: `|{{ "" | length }}|{{ "a" | length }}|{{ '' | length }}|{{ 'a' | length }}|`,
 
 	// Function calls
 	FUNCTIONS: `{{ func() }}{{ func(apple) }}{{ func(x, 'test', 2, false) }}`,
@@ -72,6 +74,8 @@ const TEST_STRINGS = {
 	FILTER_OPERATOR: `{{ arr | length }}{{ 1 + arr | length }}{{ 2 + arr | sort | length }}{{ (arr | sort)[0] }}`,
 	FILTER_OPERATOR_2: `|{{ 'abc' | length }}|{{ 'aBcD' | upper }}|{{ 'aBcD' | lower }}|{{ 'test test' | capitalize}}|{{ 'test test' | title }}|{{ ' a b ' | trim }}|{{ '  A  B  ' | trim | lower | length }}|`,
 	FILTER_OPERATOR_3: `|{{ -1 | abs }}|{{ 1 | abs }}|`,
+	FILTER_OPERATOR_4: `{{ items | selectattr('key') | length }}`,
+	FILTER_OPERATOR_5: `{{ messages | selectattr('role', 'equalto', 'system') | length }}`,
 
 	// Logical operators between non-Booleans
 	BOOLEAN_NUMERICAL: `|{{ 1 and 2 }}|{{ 1 and 0 }}|{{ 0 and 1 }}|{{ 0 and 0 }}|{{ 1 or 2 }}|{{ 1 or 0 }}|{{ 0 or 1 }}|{{ 0 or 0 }}|{{ not 1 }}|{{ not 0 }}|`,
@@ -95,6 +99,30 @@ const TEST_STRINGS = {
 	NAMESPACE: `{% set ns = namespace() %}{% set ns.foo = 'bar' %}{{ ns.foo }}`,
 	NAMESPACE_1: `{% set ns = namespace(default=false) %}{{ ns.default }}`,
 	NAMESPACE_2: `{% set ns = namespace(default=false, number=1+1) %}|{{ ns.default }}|{{ ns.number }}|`,
+
+	// Object operators
+	OBJECT_OPERATORS: `|{{ 'known' in obj }}|{{ 'known' not in obj }}|{{ 'unknown' in obj }}|{{ 'unknown' not in obj }}|`,
+	OBJECT_OPERATORS_1: `|{{ obj.get('known') }}|{{ obj.get('unknown') is none }}|{{ obj.get('unknown') is defined }}|`,
+
+	// Scope
+	SCOPE: `{% set ns = namespace(found=false) %}{% for num in nums %}{% if num == 1 %}{{ 'found=' }}{% set ns.found = true %}{% endif %}{% endfor %}{{ ns.found }}`,
+	SCOPE_1: `{% set found = false %}{% for num in nums %}{% if num == 1 %}{{ 'found=' }}{% set found = true %}{% endif %}{% endfor %}{{ found }}`,
+
+	// Undefined
+	UNDEFINED_VARIABLES: `{{ undefined_variable }}`,
+	UNDEFINED_ACCESS: `{{ object.undefined_attribute }}`,
+
+	// Ternary operator
+	TERNARY_OPERATOR: `|{{ 'a' if true else 'b' }}|{{ 'a' if false else 'b' }}|{{ 'a' if 1 + 1 == 2 else 'b' }}|{{ 'a' if 1 + 1 == 3 or 1 * 2 == 3 else 'b' }}|`,
+
+	// Array literals
+	ARRAY_LITERALS: `{{ [1, true, 'hello', [1, 2, 3, 4], var] | length }}`,
+
+	// Object literals
+	OBJECT_LITERALS: `{{ { 'key': 'value', key: 'value2', "key3": [1, {'foo': 'bar'} ] }['key'] }}`,
+
+	// Array operators
+	ARRAY_OPERATORS: `{{ ([1, 2, 3] + [4, 5, 6]) | length }}`,
 };
 
 const TEST_PARSED = {
@@ -613,6 +641,62 @@ const TEST_PARSED = {
 		{ value: "[INST] ", type: "StringLiteral" },
 		{ value: "}}", type: "CloseExpression" },
 	],
+	STRINGS_1: [
+		{ value: "|", type: "Text" },
+		{ value: "{{", type: "OpenExpression" },
+		{ value: "test", type: "StringLiteral" },
+		{ value: "}}", type: "CloseExpression" },
+		{ value: "|", type: "Text" },
+		{ value: "{{", type: "OpenExpression" },
+		{ value: "a", type: "StringLiteral" },
+		{ value: "+", type: "AdditiveBinaryOperator" },
+		{ value: "b", type: "StringLiteral" },
+		{ value: "+", type: "AdditiveBinaryOperator" },
+		{ value: "c", type: "StringLiteral" },
+		{ value: "}}", type: "CloseExpression" },
+		{ value: "|", type: "Text" },
+		{ value: "{{", type: "OpenExpression" },
+		{ value: '"', type: "StringLiteral" },
+		{ value: "+", type: "AdditiveBinaryOperator" },
+		{ value: "'", type: "StringLiteral" },
+		{ value: "}}", type: "CloseExpression" },
+		{ value: "|", type: "Text" },
+		{ value: "{{", type: "OpenExpression" },
+		{ value: "'", type: "StringLiteral" },
+		{ value: "}}", type: "CloseExpression" },
+		{ value: "|", type: "Text" },
+		{ value: "{{", type: "OpenExpression" },
+		{ value: '"', type: "StringLiteral" },
+		{ value: "}}", type: "CloseExpression" },
+		{ value: "|", type: "Text" },
+	],
+	STRINGS_2: [
+		{ value: "|", type: "Text" },
+		{ value: "{{", type: "OpenExpression" },
+		{ value: "", type: "StringLiteral" },
+		{ value: "|", type: "Pipe" },
+		{ value: "length", type: "Identifier" },
+		{ value: "}}", type: "CloseExpression" },
+		{ value: "|", type: "Text" },
+		{ value: "{{", type: "OpenExpression" },
+		{ value: "a", type: "StringLiteral" },
+		{ value: "|", type: "Pipe" },
+		{ value: "length", type: "Identifier" },
+		{ value: "}}", type: "CloseExpression" },
+		{ value: "|", type: "Text" },
+		{ value: "{{", type: "OpenExpression" },
+		{ value: "", type: "StringLiteral" },
+		{ value: "|", type: "Pipe" },
+		{ value: "length", type: "Identifier" },
+		{ value: "}}", type: "CloseExpression" },
+		{ value: "|", type: "Text" },
+		{ value: "{{", type: "OpenExpression" },
+		{ value: "a", type: "StringLiteral" },
+		{ value: "|", type: "Pipe" },
+		{ value: "length", type: "Identifier" },
+		{ value: "}}", type: "CloseExpression" },
+		{ value: "|", type: "Text" },
+	],
 
 	// Function calls
 	FUNCTIONS: [
@@ -1231,6 +1315,34 @@ const TEST_PARSED = {
 		{ value: "}}", type: "CloseExpression" },
 		{ value: "|", type: "Text" },
 	],
+	FILTER_OPERATOR_4: [
+		{ value: "{{", type: "OpenExpression" },
+		{ value: "items", type: "Identifier" },
+		{ value: "|", type: "Pipe" },
+		{ value: "selectattr", type: "Identifier" },
+		{ value: "(", type: "OpenParen" },
+		{ value: "key", type: "StringLiteral" },
+		{ value: ")", type: "CloseParen" },
+		{ value: "|", type: "Pipe" },
+		{ value: "length", type: "Identifier" },
+		{ value: "}}", type: "CloseExpression" },
+	],
+	FILTER_OPERATOR_5: [
+		{ value: "{{", type: "OpenExpression" },
+		{ value: "messages", type: "Identifier" },
+		{ value: "|", type: "Pipe" },
+		{ value: "selectattr", type: "Identifier" },
+		{ value: "(", type: "OpenParen" },
+		{ value: "role", type: "StringLiteral" },
+		{ value: ",", type: "Comma" },
+		{ value: "equalto", type: "StringLiteral" },
+		{ value: ",", type: "Comma" },
+		{ value: "system", type: "StringLiteral" },
+		{ value: ")", type: "CloseParen" },
+		{ value: "|", type: "Pipe" },
+		{ value: "length", type: "Identifier" },
+		{ value: "}}", type: "CloseExpression" },
+	],
 
 	// Logical operators between non-Booleans
 	BOOLEAN_NUMERICAL: [
@@ -1777,6 +1889,303 @@ const TEST_PARSED = {
 		{ value: "}}", type: "CloseExpression" },
 		{ value: "|", type: "Text" },
 	],
+
+	// Object operators
+	OBJECT_OPERATORS: [
+		{ value: "|", type: "Text" },
+		{ value: "{{", type: "OpenExpression" },
+		{ value: "known", type: "StringLiteral" },
+		{ value: "in", type: "In" },
+		{ value: "obj", type: "Identifier" },
+		{ value: "}}", type: "CloseExpression" },
+		{ value: "|", type: "Text" },
+		{ value: "{{", type: "OpenExpression" },
+		{ value: "known", type: "StringLiteral" },
+		{ value: "not in", type: "NotIn" },
+		{ value: "obj", type: "Identifier" },
+		{ value: "}}", type: "CloseExpression" },
+		{ value: "|", type: "Text" },
+		{ value: "{{", type: "OpenExpression" },
+		{ value: "unknown", type: "StringLiteral" },
+		{ value: "in", type: "In" },
+		{ value: "obj", type: "Identifier" },
+		{ value: "}}", type: "CloseExpression" },
+		{ value: "|", type: "Text" },
+		{ value: "{{", type: "OpenExpression" },
+		{ value: "unknown", type: "StringLiteral" },
+		{ value: "not in", type: "NotIn" },
+		{ value: "obj", type: "Identifier" },
+		{ value: "}}", type: "CloseExpression" },
+		{ value: "|", type: "Text" },
+	],
+	OBJECT_OPERATORS_1: [
+		{ value: "|", type: "Text" },
+		{ value: "{{", type: "OpenExpression" },
+		{ value: "obj", type: "Identifier" },
+		{ value: ".", type: "Dot" },
+		{ value: "get", type: "Identifier" },
+		{ value: "(", type: "OpenParen" },
+		{ value: "known", type: "StringLiteral" },
+		{ value: ")", type: "CloseParen" },
+		{ value: "}}", type: "CloseExpression" },
+		{ value: "|", type: "Text" },
+		{ value: "{{", type: "OpenExpression" },
+		{ value: "obj", type: "Identifier" },
+		{ value: ".", type: "Dot" },
+		{ value: "get", type: "Identifier" },
+		{ value: "(", type: "OpenParen" },
+		{ value: "unknown", type: "StringLiteral" },
+		{ value: ")", type: "CloseParen" },
+		{ value: "is", type: "Is" },
+		{ value: "none", type: "Identifier" },
+		{ value: "}}", type: "CloseExpression" },
+		{ value: "|", type: "Text" },
+		{ value: "{{", type: "OpenExpression" },
+		{ value: "obj", type: "Identifier" },
+		{ value: ".", type: "Dot" },
+		{ value: "get", type: "Identifier" },
+		{ value: "(", type: "OpenParen" },
+		{ value: "unknown", type: "StringLiteral" },
+		{ value: ")", type: "CloseParen" },
+		{ value: "is", type: "Is" },
+		{ value: "defined", type: "Identifier" },
+		{ value: "}}", type: "CloseExpression" },
+		{ value: "|", type: "Text" },
+	],
+
+	// Scope
+	SCOPE: [
+		{ value: "{%", type: "OpenStatement" },
+		{ value: "set", type: "Set" },
+		{ value: "ns", type: "Identifier" },
+		{ value: "=", type: "Equals" },
+		{ value: "namespace", type: "Identifier" },
+		{ value: "(", type: "OpenParen" },
+		{ value: "found", type: "Identifier" },
+		{ value: "=", type: "Equals" },
+		{ value: "false", type: "BooleanLiteral" },
+		{ value: ")", type: "CloseParen" },
+		{ value: "%}", type: "CloseStatement" },
+		{ value: "{%", type: "OpenStatement" },
+		{ value: "for", type: "For" },
+		{ value: "num", type: "Identifier" },
+		{ value: "in", type: "In" },
+		{ value: "nums", type: "Identifier" },
+		{ value: "%}", type: "CloseStatement" },
+		{ value: "{%", type: "OpenStatement" },
+		{ value: "if", type: "If" },
+		{ value: "num", type: "Identifier" },
+		{ value: "==", type: "ComparisonBinaryOperator" },
+		{ value: "1", type: "NumericLiteral" },
+		{ value: "%}", type: "CloseStatement" },
+		{ value: "{{", type: "OpenExpression" },
+		{ value: "found=", type: "StringLiteral" },
+		{ value: "}}", type: "CloseExpression" },
+		{ value: "{%", type: "OpenStatement" },
+		{ value: "set", type: "Set" },
+		{ value: "ns", type: "Identifier" },
+		{ value: ".", type: "Dot" },
+		{ value: "found", type: "Identifier" },
+		{ value: "=", type: "Equals" },
+		{ value: "true", type: "BooleanLiteral" },
+		{ value: "%}", type: "CloseStatement" },
+		{ value: "{%", type: "OpenStatement" },
+		{ value: "endif", type: "EndIf" },
+		{ value: "%}", type: "CloseStatement" },
+		{ value: "{%", type: "OpenStatement" },
+		{ value: "endfor", type: "EndFor" },
+		{ value: "%}", type: "CloseStatement" },
+		{ value: "{{", type: "OpenExpression" },
+		{ value: "ns", type: "Identifier" },
+		{ value: ".", type: "Dot" },
+		{ value: "found", type: "Identifier" },
+		{ value: "}}", type: "CloseExpression" },
+	],
+	SCOPE_1: [
+		{ value: "{%", type: "OpenStatement" },
+		{ value: "set", type: "Set" },
+		{ value: "found", type: "Identifier" },
+		{ value: "=", type: "Equals" },
+		{ value: "false", type: "BooleanLiteral" },
+		{ value: "%}", type: "CloseStatement" },
+		{ value: "{%", type: "OpenStatement" },
+		{ value: "for", type: "For" },
+		{ value: "num", type: "Identifier" },
+		{ value: "in", type: "In" },
+		{ value: "nums", type: "Identifier" },
+		{ value: "%}", type: "CloseStatement" },
+		{ value: "{%", type: "OpenStatement" },
+		{ value: "if", type: "If" },
+		{ value: "num", type: "Identifier" },
+		{ value: "==", type: "ComparisonBinaryOperator" },
+		{ value: "1", type: "NumericLiteral" },
+		{ value: "%}", type: "CloseStatement" },
+		{ value: "{{", type: "OpenExpression" },
+		{ value: "found=", type: "StringLiteral" },
+		{ value: "}}", type: "CloseExpression" },
+		{ value: "{%", type: "OpenStatement" },
+		{ value: "set", type: "Set" },
+		{ value: "found", type: "Identifier" },
+		{ value: "=", type: "Equals" },
+		{ value: "true", type: "BooleanLiteral" },
+		{ value: "%}", type: "CloseStatement" },
+		{ value: "{%", type: "OpenStatement" },
+		{ value: "endif", type: "EndIf" },
+		{ value: "%}", type: "CloseStatement" },
+		{ value: "{%", type: "OpenStatement" },
+		{ value: "endfor", type: "EndFor" },
+		{ value: "%}", type: "CloseStatement" },
+		{ value: "{{", type: "OpenExpression" },
+		{ value: "found", type: "Identifier" },
+		{ value: "}}", type: "CloseExpression" },
+	],
+
+	// Undefined
+	UNDEFINED_VARIABLES: [
+		{ value: "{{", type: "OpenExpression" },
+		{ value: "undefined_variable", type: "Identifier" },
+		{ value: "}}", type: "CloseExpression" },
+	],
+	UNDEFINED_ACCESS: [
+		{ value: "{{", type: "OpenExpression" },
+		{ value: "object", type: "Identifier" },
+		{ value: ".", type: "Dot" },
+		{ value: "undefined_attribute", type: "Identifier" },
+		{ value: "}}", type: "CloseExpression" },
+	],
+
+	// Ternary operator
+	TERNARY_OPERATOR: [
+		{ value: "|", type: "Text" },
+		{ value: "{{", type: "OpenExpression" },
+		{ value: "a", type: "StringLiteral" },
+		{ value: "if", type: "If" },
+		{ value: "true", type: "BooleanLiteral" },
+		{ value: "else", type: "Else" },
+		{ value: "b", type: "StringLiteral" },
+		{ value: "}}", type: "CloseExpression" },
+		{ value: "|", type: "Text" },
+		{ value: "{{", type: "OpenExpression" },
+		{ value: "a", type: "StringLiteral" },
+		{ value: "if", type: "If" },
+		{ value: "false", type: "BooleanLiteral" },
+		{ value: "else", type: "Else" },
+		{ value: "b", type: "StringLiteral" },
+		{ value: "}}", type: "CloseExpression" },
+		{ value: "|", type: "Text" },
+		{ value: "{{", type: "OpenExpression" },
+		{ value: "a", type: "StringLiteral" },
+		{ value: "if", type: "If" },
+		{ value: "1", type: "NumericLiteral" },
+		{ value: "+", type: "AdditiveBinaryOperator" },
+		{ value: "1", type: "NumericLiteral" },
+		{ value: "==", type: "ComparisonBinaryOperator" },
+		{ value: "2", type: "NumericLiteral" },
+		{ value: "else", type: "Else" },
+		{ value: "b", type: "StringLiteral" },
+		{ value: "}}", type: "CloseExpression" },
+		{ value: "|", type: "Text" },
+		{ value: "{{", type: "OpenExpression" },
+		{ value: "a", type: "StringLiteral" },
+		{ value: "if", type: "If" },
+		{ value: "1", type: "NumericLiteral" },
+		{ value: "+", type: "AdditiveBinaryOperator" },
+		{ value: "1", type: "NumericLiteral" },
+		{ value: "==", type: "ComparisonBinaryOperator" },
+		{ value: "3", type: "NumericLiteral" },
+		{ value: "or", type: "Or" },
+		{ value: "1", type: "NumericLiteral" },
+		{ value: "*", type: "MultiplicativeBinaryOperator" },
+		{ value: "2", type: "NumericLiteral" },
+		{ value: "==", type: "ComparisonBinaryOperator" },
+		{ value: "3", type: "NumericLiteral" },
+		{ value: "else", type: "Else" },
+		{ value: "b", type: "StringLiteral" },
+		{ value: "}}", type: "CloseExpression" },
+		{ value: "|", type: "Text" },
+	],
+
+	// Array literals
+	ARRAY_LITERALS: [
+		{ value: "{{", type: "OpenExpression" },
+		{ value: "[", type: "OpenSquareBracket" },
+		{ value: "1", type: "NumericLiteral" },
+		{ value: ",", type: "Comma" },
+		{ value: "true", type: "BooleanLiteral" },
+		{ value: ",", type: "Comma" },
+		{ value: "hello", type: "StringLiteral" },
+		{ value: ",", type: "Comma" },
+		{ value: "[", type: "OpenSquareBracket" },
+		{ value: "1", type: "NumericLiteral" },
+		{ value: ",", type: "Comma" },
+		{ value: "2", type: "NumericLiteral" },
+		{ value: ",", type: "Comma" },
+		{ value: "3", type: "NumericLiteral" },
+		{ value: ",", type: "Comma" },
+		{ value: "4", type: "NumericLiteral" },
+		{ value: "]", type: "CloseSquareBracket" },
+		{ value: ",", type: "Comma" },
+		{ value: "var", type: "Identifier" },
+		{ value: "]", type: "CloseSquareBracket" },
+		{ value: "|", type: "Pipe" },
+		{ value: "length", type: "Identifier" },
+		{ value: "}}", type: "CloseExpression" },
+	],
+
+	// Object literals
+	OBJECT_LITERALS: [
+		{ value: "{{", type: "OpenExpression" },
+		{ value: "{", type: "OpenCurlyBracket" },
+		{ value: "key", type: "StringLiteral" },
+		{ value: ":", type: "Colon" },
+		{ value: "value", type: "StringLiteral" },
+		{ value: ",", type: "Comma" },
+		{ value: "key", type: "Identifier" },
+		{ value: ":", type: "Colon" },
+		{ value: "value2", type: "StringLiteral" },
+		{ value: ",", type: "Comma" },
+		{ value: "key3", type: "StringLiteral" },
+		{ value: ":", type: "Colon" },
+		{ value: "[", type: "OpenSquareBracket" },
+		{ value: "1", type: "NumericLiteral" },
+		{ value: ",", type: "Comma" },
+		{ value: "{", type: "OpenCurlyBracket" },
+		{ value: "foo", type: "StringLiteral" },
+		{ value: ":", type: "Colon" },
+		{ value: "bar", type: "StringLiteral" },
+		{ value: "}", type: "CloseCurlyBracket" },
+		{ value: "]", type: "CloseSquareBracket" },
+		{ value: "}", type: "CloseCurlyBracket" },
+		{ value: "[", type: "OpenSquareBracket" },
+		{ value: "key", type: "StringLiteral" },
+		{ value: "]", type: "CloseSquareBracket" },
+		{ value: "}}", type: "CloseExpression" },
+	],
+
+	// Array operators
+	ARRAY_OPERATORS: [
+		{ value: "{{", type: "OpenExpression" },
+		{ value: "(", type: "OpenParen" },
+		{ value: "[", type: "OpenSquareBracket" },
+		{ value: "1", type: "NumericLiteral" },
+		{ value: ",", type: "Comma" },
+		{ value: "2", type: "NumericLiteral" },
+		{ value: ",", type: "Comma" },
+		{ value: "3", type: "NumericLiteral" },
+		{ value: "]", type: "CloseSquareBracket" },
+		{ value: "+", type: "AdditiveBinaryOperator" },
+		{ value: "[", type: "OpenSquareBracket" },
+		{ value: "4", type: "NumericLiteral" },
+		{ value: ",", type: "Comma" },
+		{ value: "5", type: "NumericLiteral" },
+		{ value: ",", type: "Comma" },
+		{ value: "6", type: "NumericLiteral" },
+		{ value: "]", type: "CloseSquareBracket" },
+		{ value: ")", type: "CloseParen" },
+		{ value: "|", type: "Pipe" },
+		{ value: "length", type: "Identifier" },
+		{ value: "}}", type: "CloseExpression" },
+	],
 };
 
 const TEST_CONTEXT = {
@@ -1825,6 +2234,8 @@ const TEST_CONTEXT = {
 	STRINGS: {
 		bos_token: "<s>",
 	},
+	STRINGS_1: {},
+	STRINGS_2: {},
 
 	// Function calls
 	FUNCTIONS: {
@@ -1887,6 +2298,12 @@ const TEST_CONTEXT = {
 	},
 	FILTER_OPERATOR_2: {},
 	FILTER_OPERATOR_3: {},
+	FILTER_OPERATOR_4: {
+		items: [{ key: "a" }, { key: 0 }, { key: 1 }, {}, { key: false }],
+	},
+	FILTER_OPERATOR_5: {
+		messages: [{ role: "system" }, { role: "user" }, { role: "assistant" }],
+	},
 
 	// Logical operators between non-Booleans
 	BOOLEAN_NUMERICAL: {},
@@ -1914,6 +2331,40 @@ const TEST_CONTEXT = {
 	NAMESPACE: {},
 	NAMESPACE_1: {},
 	NAMESPACE_2: {},
+
+	// Object operators
+	OBJECT_OPERATORS: {
+		obj: {
+			known: true,
+		},
+	},
+	OBJECT_OPERATORS_1: {
+		obj: {
+			known: true,
+		},
+	},
+
+	// Scope
+	SCOPE: { nums: [1, 2, 3] },
+	SCOPE_1: { nums: [1, 2, 3] },
+
+	// Undefined
+	UNDEFINED_VARIABLES: {},
+	UNDEFINED_ACCESS: { object: {} },
+
+	// Ternary operator
+	TERNARY_OPERATOR: {},
+
+	// Array literals
+	ARRAY_LITERALS: { var: true },
+
+	// Object literals
+	OBJECT_LITERALS: {
+		key: "key2",
+	},
+
+	// Array operators
+	ARRAY_OPERATORS: {},
 };
 
 const EXPECTED_OUTPUTS = {
@@ -1951,6 +2402,8 @@ const EXPECTED_OUTPUTS = {
 
 	// Strings
 	STRINGS: "Bye<s>[INST] ",
+	STRINGS_1: `|test|abc|"'|'|"|`,
+	STRINGS_2: `|0|1|0|1|`,
 
 	// Function calls
 	FUNCTIONS: "014",
@@ -1984,6 +2437,8 @@ const EXPECTED_OUTPUTS = {
 	FILTER_OPERATOR: `3451`,
 	FILTER_OPERATOR_2: `|3|ABCD|abcd|Test test|Test Test|a b|4|`,
 	FILTER_OPERATOR_3: `|1|1|`,
+	FILTER_OPERATOR_4: `2`,
+	FILTER_OPERATOR_5: `1`,
 
 	// Logical operators between non-Booleans
 	BOOLEAN_NUMERICAL: `|2|0|0|0|1|1|1|0|false|true|`,
@@ -2007,6 +2462,30 @@ const EXPECTED_OUTPUTS = {
 	NAMESPACE: `bar`,
 	NAMESPACE_1: `false`,
 	NAMESPACE_2: `|false|2|`,
+
+	// Object operators
+	OBJECT_OPERATORS: `|true|false|false|true|`,
+	OBJECT_OPERATORS_1: `|true|true|true|`,
+
+	// Scope
+	SCOPE: `found=true`,
+	SCOPE_1: `found=false`,
+
+	// Undefined
+	UNDEFINED_VARIABLES: ``,
+	UNDEFINED_ACCESS: ``,
+
+	// Ternary operator
+	TERNARY_OPERATOR: `|a|b|a|b|`,
+
+	// Array literals
+	ARRAY_LITERALS: `5`,
+
+	// Object literals
+	OBJECT_LITERALS: `value`,
+
+	// Array operators
+	ARRAY_OPERATORS: `6`,
 };
 
 describe("Templates", () => {
@@ -2080,6 +2559,11 @@ describe("Error checking", () => {
 			const text = "{{ invalid ! invalid }}";
 			expect(() => tokenize(text)).toThrowError();
 		});
+
+		it("Invalid quote character", () => {
+			const text = "{{ \u2018text\u2019 }}";
+			expect(() => tokenize(text)).toThrowError();
+		});
 	});
 
 	describe("Parsing errors", () => {
@@ -2127,22 +2611,6 @@ describe("Error checking", () => {
 	});
 
 	describe("Runtime errors", () => {
-		it("Undefined variable", () => {
-			const env = new Environment();
-			const interpreter = new Interpreter(env);
-			const tokens = tokenize("{{ undefined_variable }}");
-			const ast = parse(tokens);
-			expect(() => interpreter.run(ast)).toThrowError();
-		});
-
-		it("Undefined attribute access", () => {
-			const env = new Environment();
-			const interpreter = new Interpreter(env);
-			const tokens = tokenize("{{ object.undefined_attribute }}");
-			const ast = parse(tokens);
-			expect(() => interpreter.run(ast)).toThrowError();
-		});
-
 		it("Undefined function call", () => {
 			const env = new Environment();
 			const interpreter = new Interpreter(env);

packages/widgets/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@huggingface/widgets",
 	"packageManager": "[email protected]",
-	"version": "0.2.3",
+	"version": "0.2.4",
 	"publishConfig": {
 		"access": "public"
 	},