:- module lexer.
:- use_module builtin, char, int, io, list, private_builtin, require, string, term.
:- type (lexer:string_token_context) == int.
lexer:string_get_token_list(String_5, Tokens_6, DCG_0_8, DCG_1_9) :-
		string:length(String_5, Len_7),
		lexer:string_get_token_list(String_5, Len_7, Tokens_6, DCG_0_8, DCG_1_9).
:- pragma termination_info(lexer:get_token_list((builtin:out), (builtin:di), (builtin:uo)), infinite, can_loop).
:- pragma termination_info(lexer:string_get_token_list((builtin:in), (builtin:in), (builtin:out), (builtin:in), (builtin:out)), infinite, can_loop).
:- pragma termination_info(lexer:string_get_token_list((builtin:in), (builtin:out), (builtin:in), (builtin:out)), infinite, can_loop).
:- pragma termination_info(lexer:token_to_string((builtin:in), (builtin:out)), infinite, can_loop).
:- pragma termination_info(lexer:graphic_token_char((builtin:in)), finite(0, [no]), cannot_loop).