lex_by_file( FileName ) :-
get_input_from_file( FileName, TokenList ),
lex( TokenList, OutputList ),
%write_output( OutputList ), !.
maplist(writeln,OutputList),!.
%% atom_number(atom, number)
lex([],[]).
lex(['int' | T], ['TYPE: int' | R]) :- lex(T, R).
lex(['bool' | T], ['TYPE: bool'] | R) :- lex(T, R).
lex([',' | T], ['COMMA: ,'] | R) :- lex(T, R).
lex([X | T], [X | R]) :- lex(T, R).
bGV4X2J5X2ZpbGUoIEZpbGVOYW1lICkgOi0KCWdldF9pbnB1dF9mcm9tX2ZpbGUoIEZpbGVOYW1lLCBUb2tlbkxpc3QgKSwKCWxleCggVG9rZW5MaXN0LCBPdXRwdXRMaXN0ICksCgkld3JpdGVfb3V0cHV0KCBPdXRwdXRMaXN0ICksICEuCgltYXBsaXN0KHdyaXRlbG4sT3V0cHV0TGlzdCksIS4KCiUlIGF0b21fbnVtYmVyKGF0b20sIG51bWJlcikKbGV4KFtdLFtdKS4KbGV4KFsnaW50JyB8IFRdLCBbJ1RZUEU6IGludCcgfCBSXSkgOi0gIGxleChULCBSKS4KbGV4KFsnYm9vbCcgfCBUXSwgWydUWVBFOiBib29sJ10gfCBSKSA6LSBsZXgoVCwgUikuCmxleChbJywnIHwgVF0sIFsnQ09NTUE6ICwnXSB8IFIpIDotIGxleChULCBSKS4KbGV4KFtYIHwgVF0sIFtYIHwgUl0pIDotIGxleChULCBSKS4K
I2luY2x1ZGUgJmx0O3N0ZGlvLmgmZ3Q7CiNpbmNsdWRlICZsdDtzdHJpbmcuaCZndDsKCmludCBtYWluKCkgewogICAgY2hhciBzdHJbXSA9ICZxdW90O1RoaXMgaXMgYSBzYW1wbGUgc3RyaW5nIGZvciB0b2tlbml6YXRpb24mcXVvdDs7CiAgICBjaGFyICp0b2tlbjsKCiAgICAvLyBGaXJzdCBjYWxsIHRvIHN0cnRvaygpCiAgICB0b2tlbiA9IHN0cnRvayhzdHIsICZxdW90OyAmcXVvdDspOwoKICAgIC8vIFN1YnNlcXVlbnQgY2FsbHMgdG8gc3RydG9rKCkKICAgIHdoaWxlICh0b2tlbiAhPSBOVUxMKSB7CiAgICAgICAgcHJpbnRmKCZxdW90OyVzXG4mcXVvdDssIHRva2VuKTsKICAgICAgICB0b2tlbiA9IHN0cnRvayhOVUxMLCAmcXVvdDsgJnF1b3Q7KTsKICAgIH0KCiAgICByZXR1cm4gMDsKfQ==
#include <stdio.h>
#include <string.h>
int main() {
char str[] = "This is a sample string for tokenization";
char *token;
// First call to strtok()
token = strtok(str, " ");
// Subsequent calls to strtok()
while (token != NULL) {
printf("%s\n", token);
token = strtok(NULL, " ");
}
return 0;
}