we now strip characters in strings

lab-5/Makefile

@@ -2,14 +2,16 @@
 # The top level rule indicates how to link everything together into main
 
 main: main.o symbol_map.o expr.o scanner.o parser.o
-	gcc main.o symbol_map.o expr.o scanner.o parser.o -o interpreter.out -lm
+	gcc -g3 -Wall main.o symbol_map.o expr.o scanner.o parser.o -o interpreter.out -lm
 
 test: main_test.o symbol_map.o expr.o scanner.o parser.o
-	gcc main_test.o symbol_map.o expr.o scanner.o parser.o -o interpreter_test.out -lm
+	gcc -g3 -Wall main_test.o symbol_map.o expr.o scanner.o parser.o -o interpreter_test.out -lm
 
 parser_test: parser_test.o symbol_map.o expr.o scanner.o parser.o
-	gcc parser_test.o symbol_map.o expr.o scanner.o parser.o -o parser_test.out -lm
+	gcc -g3 -Wall parser_test.o symbol_map.o expr.o scanner.o parser.o -o parser_test.out -lm
 
+scanner_test: scanner_test.o symbol_map.o expr.o scanner.o parser.o
+	gcc -g3 -Wall scanner_test.o symbol_map.o expr.o scanner.o parser.o -o scanner_test.out -lm
 
 run_parser_test: parser_test
 	./parser_test.out
@@ -17,7 +19,10 @@ run_parser_test: parser_test
 run_interpreter_test: test
 	./interpreter_test.out
 
-run_test: run_parser_test run_interpreter_test
+run_scanner_test: scanner_test
+	./scanner_test.out
+
+run_tests: run_parser_test run_interpreter_test run_scanner_test
 
 # This pattern indicates that any .o file depends
 # upon the .c file of the same name, and all of the .h files.

lab-5/expr.c

@@ -1,4 +1,3 @@
-
 #include "expr.h"
 
 #include <stdio.h>
@@ -35,7 +34,6 @@ struct stmt *stmt_create(stmt_t kind, struct decl *decl, struct expr *init_expr,
                          struct stmt *body, struct stmt *else_body,
                          struct stmt *next) {
   struct stmt *s = malloc(sizeof(*s));
-
   s->kind = kind;
   s->decl = decl;
   s->init_expr = init_expr;
@@ -44,7 +42,6 @@ struct stmt *stmt_create(stmt_t kind, struct decl *decl, struct expr *init_expr,
   s->body = body;
   s->else_body = else_body;
   s->next = next;
-
   return s;
 }
 
@@ -85,8 +82,8 @@ struct expr *expr_create_float_literal(float value) {
 
 struct expr *expr_create_string_literal(const char *value) {
   struct expr *e = expr_create(EXPR_STRING_LITERAL, 0, 0);
-  char *dest = malloc(sizeof(*value));
+  char *dest = malloc(strlen(value) + 1);
-  strcpy(dest, value); // copy contents of source to dest
+  strcpy(dest, value);
   e->string_literal = dest;
   return e;
 }

lab-5/parser.bison

@@ -96,9 +96,9 @@ extern struct stmt *parser_result = 0;
 }
 
 %type <stmt> statement_list statement section stop_run sect_data simple_stmt cbl_func_stmt cbl_function if_branch else_parts perform_stmt data_space
-%type <expr> mathmaticalexpr booleanexpr term op_parm container_expr type_expr op_parms math_op categry_contain category_value
+%type <expr> mathmaticalexpr booleanexpr term op_parm container_expr type_expr op_parms math_op categry_contain category_value ident
-%type <decl> assignment_stmt simple_decl complex_decl data_declaration
+%type <decl> assignment_stmt simple_decl complex_decl data_declaration category_spec
-%type <type> data_category complete_category category_spec data_clause
+%type <type> data_category complete_category data_clause
 %%
 file            : statement_list
                     {parser_result = $1; return 0;}
@@ -127,7 +127,7 @@ section         : type TOKEN_KEYWORD_DIVISION TOKEN_DOT
 stop_run        : TOKEN_STOP TOKEN_RUN TOKEN_DOT
                     {$$ = stmt_create(STMT_END_EXECUTION, NULL, NULL, NULL, NULL, NULL, NULL, NULL);}
                 ;
-sect_data       : TOKEN_PROGRAM_ID TOKEN_DOT TOKEN_IDENT TOKEN_DOT
+sect_data       : TOKEN_PROGRAM_ID TOKEN_DOT ident TOKEN_DOT
                     {$$ = stmt_create(STMT_SECTION, NULL, NULL, NULL, NULL, NULL, NULL, NULL);}
                 ;
 type            : TOKEN_KEYWORD_IDENTIFICATION
@@ -194,7 +194,7 @@ booleanexpr     : mathmaticalexpr TOKEN_LESS_THAN term
                 | mathmaticalexpr TOKEN_EQUAL term
                     {$$ = expr_create(EXPR_EQUAL_EQUAL, $1, $3);}
                 ;
-type_expr       : TOKEN_IDENT
+type_expr       : ident
                     {$$ = expr_create_name(yytext);}
                 | TOKEN_INTEGER
                     {$$ = expr_create_integer_literal(atoi(yytext));}
@@ -203,10 +203,10 @@ type_expr       : TOKEN_IDENT
                 | TOKEN_SPACE
                     {$$ = expr_create_integer_literal(0);}
                 // TODO: implment negative numbers
-                | TOKEN_SUB TOKEN_IDENT
+                | TOKEN_SUB ident
                 | ext_function
                 ;
-ext_function    : TOKEN_KEYWORD_FUNCTION TOKEN_IDENT TOKEN_LEFT_PARENTHESIS TOKEN_IDENT TOKEN_RIGHT_PARENTHESIS
+ext_function    : TOKEN_KEYWORD_FUNCTION ident TOKEN_LEFT_PARENTHESIS ident TOKEN_RIGHT_PARENTHESIS
 
                 ;
 cbl_function    : TOKEN_DISPLAY
@@ -217,13 +217,15 @@ cbl_function    : TOKEN_DISPLAY
                     {$$ = stmt_create(STMT_COMPUTE, NULL, NULL, NULL, NULL, NULL, NULL, NULL);}
                 ;
 if_branch       : TOKEN_IF booleanexpr statement_list else_parts
-                    {$$ = stmt_create(STMT_IF, NULL, $2, NULL, $4, $3, NULL, NULL);}
+                    {$$ = stmt_create(STMT_IF, NULL, NULL, $2, $4, $3, NULL, NULL);}
                 ;
-else_parts      : TOKEN_ELSE_IF booleanexpr simple_stmt
+else_parts      : TOKEN_ELSE_IF booleanexpr statement_list
+                    {$$ = stmt_create(STMT_IF, NULL, NULL, $2, $3, NULL, NULL, NULL);}
                 | TOKEN_ELSE simple_stmt
+                    {$$ = stmt_create(STMT_IF, NULL, NULL, NULL, NULL, $2, NULL, NULL);}
                 | TOKEN_END_IF
                 ;
-perform_stmt    : TOKEN_PERFORM TOKEN_VARYING TOKEN_IDENT TOKEN_KEYWORD_FROM TOKEN_INTEGER TOKEN_KEYWORD_BY TOKEN_INTEGER TOKEN_UNTIL op_parms
+perform_stmt    : TOKEN_PERFORM TOKEN_VARYING ident TOKEN_KEYWORD_FROM TOKEN_INTEGER TOKEN_KEYWORD_BY TOKEN_INTEGER TOKEN_UNTIL op_parms
                 | TOKEN_END_PERFORM
                 ;
 data_space      : TOKEN_WORKING_STORAGE TOKEN_KEYWORD_SECTION TOKEN_DOT
@@ -256,11 +258,11 @@ data_clause     : TOKEN_COMPUTATION_LEVEL_0
                 |
                 ;
                 //TODO: implement levels
-simple_decl     : TOKEN_INTEGER TOKEN_IDENT TOKEN_DOT
+simple_decl     : TOKEN_INTEGER ident TOKEN_DOT
-                    {$$ = decl_create(expr_create_name(yytext), NULL, NULL, NULL, NULL);}
+                    {$$ = decl_create($2, NULL, NULL, NULL, NULL);}
                 ;
-complex_decl    : TOKEN_INTEGER TOKEN_IDENT TOKEN_PICTURE category_spec TOKEN_DOT
+complex_decl    : TOKEN_INTEGER ident TOKEN_PICTURE category_spec TOKEN_DOT
-                    {$$ = decl_create(expr_create_name(yytext), $4, NULL, NULL, NULL);}
+                    {$$ = decl_create($2, $4->type, $4->value, NULL, NULL);}
                 ;
 category_value  : TOKEN_KEYWORD_VALUE TOKEN_INTEGER
                     {$$ = expr_create_integer_literal(atoi(yytext));}
@@ -271,13 +273,16 @@ category_value  : TOKEN_KEYWORD_VALUE TOKEN_INTEGER
 category_spec   : complete_category
                     {$$ = decl_create(NULL, $1, NULL, NULL, NULL);}
                 | complete_category data_clause category_value
-                    { $$ = decl_create(NULL, $1, NULL, NULL, $2);}
+                    { $$ = decl_create(NULL, $1, $3, NULL, $2);}
                 ;
 data_declaration: simple_decl
                     {$$ = $1;}
                 | complex_decl
                     {$$ = $1;}
                 ;
+ident           : TOKEN_IDENT
+                { $$ = expr_create_name(yytext);}
+                ;
 %%
 void yyerror(const char* msg) {
     fprintf(stderr, "Error | Line: %d\n%s\n",yylineno,msg);

lab-5/samples/hello-world-multiple-args.cbl

@@ -1,5 +1,5 @@
 IDENTIFICATION DIVISION.
 PROGRAM-ID. HELLO-WORLD.
 PROCEDURE DIVISION.
-    DISPLAY 'Hello World!' 'Hello World' 'Lorem Ipsum'
+    DISPLAY 'Hello World!' ' ' 'Hello World' ' ' 'Lorem Ipsum'
 STOP RUN.

lab-5/samples/sorting-snippet.cbl

@@ -39,6 +39,7 @@ PROCEDURE DIVISION.
                 MOVE WS-SORT-ROW(WS-J) TO WS-TEMP-ROW
                 MOVE WS-SORT-ROW(WS-J + 1) TO WS-SORT-ROW(WS-J)
                 MOVE WS-TEMP-ROW TO WS-SORT-ROW(WS-J + 1)
+            END-IF
         END-PERFORM
     END-PERFORM
 
@@ -48,7 +49,7 @@ PROCEDURE DIVISION.
     PERFORM VARYING WS-INDEX FROM 1 BY 1
         UNTIL WS-INDEX > WS-SORT-MAX
         DISPLAY "Element " WS-INDEX ": " WS-SORT-ROW(WS-INDEX)
-    END-PERFORM.
+    END-PERFORM
 
     STOP RUN.
     

lab-5/scanner.flex

@@ -1,5 +1,7 @@
 %{
 #include "token.h"
+
+void stripStringChar();
 %}
 %option warn
 %option nodefault
@@ -57,8 +59,8 @@ COMP-3 { return TOKEN_COMPUTATION_LEVEL_3; }
 \< { return TOKEN_LESS_THAN; }
 \= { return TOKEN_EQUAL;}
 
-"\""[^"]*"\""   { return TOKEN_STRING; }
+\"[^"]*\" { stripStringChar(); return TOKEN_STRING; }
-"\'"[^']*"\'"   { return TOKEN_STRING; }
+\'[^']*\' { stripStringChar(); return TOKEN_STRING; }
 "("             { return TOKEN_LEFT_PARENTHESIS; }
 ")"             { return TOKEN_RIGHT_PARENTHESIS; }
 {NAME} { return TOKEN_IDENT; }
@@ -67,3 +69,21 @@ COMP-3 { return TOKEN_COMPUTATION_LEVEL_3; }
 \. { return TOKEN_DOT; }
 %%
 int yywrap() { return 1; }
+
+void stripStringChar() {
+	int length = strlen(yytext);
+	int subLength = 1;
+	int i, j;
+	for (i = 1; i < length; i++) {
+		yytext[i - subLength] = yytext[i];
+		/*if (yytext[i] == '\\') {
+			if (nextIsEscapable(&yytext[i], subLength)) {
+				i += 1;
+				subLength += 1;
+			}
+		}*/
+	}
+	for (j = length - 1; j >= length - subLength - 1; j--) {
+		yytext[j] = 0;
+	}
+}

lab-5/scanner_test.c

@@ -0,0 +1,577 @@
+#include "token.h"
+// https://github.com/sheredom/utest.h/blob/master/utest.h
+#include "utest.h"
+#include <stdio.h>
+
+typedef struct yy_buffer_state *YY_BUFFER_STATE;
+extern void yyrestart(FILE * input_file);
+extern YY_BUFFER_STATE yy_scan_buffer(char *str, int i);
+extern void yy_delete_buffer(YY_BUFFER_STATE buffer);
+extern FILE *yyin;
+extern int yylex();
+extern char *yytext;
+
+UTEST_MAIN();
+
+struct token_st {
+  yytoken_kind_t t;
+  char *p;
+};
+
+UTEST(scanner, hello) {
+  struct token_st tokens[] = {
+    {TOKEN_KEYWORD_IDENTIFICATION, "IDENTIFICATION"},
+    {TOKEN_KEYWORD_DIVISION, "DIVISION"},
+    {TOKEN_DOT, "."},
+    {TOKEN_PROGRAM_ID, "PROGRAM-ID"},
+    {TOKEN_DOT, "."}, 
+    {TOKEN_IDENT, "HELLO-WORLD"},
+    {TOKEN_DOT, "."},
+    {TOKEN_PROCEDURE, "PROCEDURE"},
+    {TOKEN_KEYWORD_DIVISION, "DIVISION"},
+    {TOKEN_DOT, "."},
+    {TOKEN_DISPLAY, "DISPLAY"},
+    {TOKEN_STRING, "'Hello World!'"},
+    {TOKEN_STOP, "STOP"},
+    {TOKEN_RUN, "RUN"},
+    {TOKEN_DOT, "."},
+    {TOKEN_EOF, ""},
+  };
+
+
+  yyin = fopen("samples/hello-world.cbl", "r");
+  ASSERT_TRUE(yyin);
+  int index = 0;
+  yytoken_kind_t t;
+  do {
+    printf("index: %d token: %d text: %s\n", index, t, yytext);
+    ASSERT_EQ(tokens[index].t, (t = yylex()));
+    ASSERT_STREQ(tokens[index].p, yytext);
+    ++index;
+  } while (t != TOKEN_EOF);
+}
+
+UTEST(scanner, quadratic) {
+  struct token_st tokens[] = {
+    {TOKEN_KEYWORD_IDENTIFICATION, "IDENTIFICATION"},
+    {TOKEN_KEYWORD_DIVISION, "DIVISION"},
+    {TOKEN_DOT, "."},
+    {TOKEN_PROGRAM_ID, "PROGRAM-ID"},
+    {TOKEN_DOT, "."}, 
+    {TOKEN_IDENT, "QuadraticSolver"},
+    {TOKEN_DOT, "."},
+    {TOKEN_KEYWORD_DATA, "DATA"},
+    {TOKEN_KEYWORD_DIVISION, "DIVISION"},
+    {TOKEN_DOT, "."},
+
+    {TOKEN_WORKING_STORAGE, "WORKING-STORAGE"},
+    {TOKEN_KEYWORD_SECTION, "SECTION"},
+    {TOKEN_DOT, "."},
+
+    {TOKEN_INTEGER, "77"},
+    {TOKEN_IDENT, "a"},
+    {TOKEN_PICTURE, "PIC"},
+    {TOKEN_SIGNED_NUMERIC, "S9"}, // Check later
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_INTEGER, "5"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_IMPLIED_DECIMAL, "V9"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_INTEGER, "5"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_COMPUTATION_LEVEL_3, "COMP-3"},
+    {TOKEN_KEYWORD_VALUE, "VALUE"},
+    {TOKEN_INTEGER, "1"},
+    {TOKEN_DOT, "."},
+
+    {TOKEN_INTEGER, "77"},
+    {TOKEN_IDENT, "b"},
+    {TOKEN_PICTURE, "PIC"},
+    {TOKEN_SIGNED_NUMERIC, "S9"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_INTEGER, "5"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_IMPLIED_DECIMAL, "V9"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_INTEGER, "5"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_COMPUTATION_LEVEL_3, "COMP-3"},
+    {TOKEN_KEYWORD_VALUE, "VALUE"},
+    {TOKEN_INTEGER, "5"},
+    {TOKEN_DOT, "."},
+
+    {TOKEN_INTEGER, "77"},
+    {TOKEN_IDENT, "c"},
+    {TOKEN_PICTURE, "PIC"},
+    {TOKEN_SIGNED_NUMERIC, "S9"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_INTEGER, "5"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_IMPLIED_DECIMAL, "V9"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_INTEGER, "5"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_COMPUTATION_LEVEL_3, "COMP-3"},
+    {TOKEN_KEYWORD_VALUE, "VALUE"},
+    {TOKEN_INTEGER, "6"},
+    {TOKEN_DOT, "."},
+
+    {TOKEN_INTEGER, "77"},
+    {TOKEN_IDENT, "discriminant"},
+    {TOKEN_PICTURE, "PIC"},
+    {TOKEN_SIGNED_NUMERIC, "S9"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_INTEGER, "5"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_IMPLIED_DECIMAL, "V9"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_INTEGER, "5"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_COMPUTATION_LEVEL_3, "COMP-3"},
+    {TOKEN_DOT, "."},
+
+    {TOKEN_INTEGER, "77"},
+    {TOKEN_IDENT, "root1"},
+    {TOKEN_PICTURE, "PIC"},
+    {TOKEN_SIGNED_NUMERIC, "S9"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_INTEGER, "5"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_IMPLIED_DECIMAL, "V9"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_INTEGER, "5"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_COMPUTATION_LEVEL_3, "COMP-3"},
+    {TOKEN_DOT, "."},
+
+    {TOKEN_INTEGER, "77"},
+    {TOKEN_IDENT, "root2"},
+    {TOKEN_PICTURE, "PIC"},
+    {TOKEN_SIGNED_NUMERIC, "S9"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_INTEGER, "5"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_IMPLIED_DECIMAL, "V9"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_INTEGER, "5"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_COMPUTATION_LEVEL_3, "COMP-3"},
+    {TOKEN_DOT, "."},
+
+    {TOKEN_INTEGER, "77"},
+    {TOKEN_IDENT, "square-root-discriminant"},
+    {TOKEN_PICTURE, "PIC"},
+    {TOKEN_SIGNED_NUMERIC, "S9"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_INTEGER, "5"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_IMPLIED_DECIMAL, "V9"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_INTEGER, "5"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_COMPUTATION_LEVEL_3, "COMP-3"},
+    {TOKEN_DOT, "."},
+
+    // {TOKEN_EOF, ""},
+
+    {TOKEN_PROCEDURE, "PROCEDURE"},
+    {TOKEN_KEYWORD_DIVISION, "DIVISION"},
+    {TOKEN_DOT, "."},
+    {TOKEN_DISPLAY, "DISPLAY"},
+    {TOKEN_STRING, "\"EQUATION: (1x^2) + 5x + 6 = 0\""},
+    {TOKEN_KEYWORD_COMPUTE, "COMPUTE"},
+    {TOKEN_IDENT, "discriminant"},
+    {TOKEN_EQUAL, "="},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_IDENT, "b"},
+    {TOKEN_EXPONENTIAL, "**"},
+    {TOKEN_INTEGER, "2"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_SUB, "-"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_INTEGER, "4"},
+    {TOKEN_MULTIPLY, "*"},
+    {TOKEN_IDENT, "a"},
+    {TOKEN_MULTIPLY, "*"},
+    {TOKEN_IDENT, "c"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+
+    // {TOKEN_EOF, ""},
+
+    {TOKEN_IF, "IF"},
+    {TOKEN_IDENT, "discriminant"},
+    {TOKEN_GREATER_THAN, ">"},
+    {TOKEN_INTEGER, "0"},
+
+    {TOKEN_KEYWORD_COMPUTE, "COMPUTE"},
+    {TOKEN_IDENT, "square-root-discriminant"},
+    {TOKEN_EQUAL, "="},
+    {TOKEN_KEYWORD_FUNCTION, "FUNCTION"},
+    {TOKEN_IDENT, "SQRT"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_IDENT, "discriminant"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+
+
+    {TOKEN_KEYWORD_COMPUTE, "COMPUTE"},
+    {TOKEN_IDENT, "root1"},
+    {TOKEN_EQUAL, "="},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_SUB, "-"},
+    {TOKEN_IDENT, "b"},
+    {TOKEN_ADD, "+"},
+    {TOKEN_IDENT, "square-root-discriminant"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_DIVIDE, "/"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_INTEGER, "2"},
+    {TOKEN_MULTIPLY, "*"},
+    {TOKEN_IDENT, "a"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+
+    {TOKEN_KEYWORD_COMPUTE, "COMPUTE"},
+    {TOKEN_IDENT, "root2"},
+    {TOKEN_EQUAL, "="},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_SUB, "-"},
+    {TOKEN_IDENT, "b"},
+    {TOKEN_SUB, "-"},
+    {TOKEN_IDENT, "square-root-discriminant"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_DIVIDE, "/"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_INTEGER, "2"},
+    {TOKEN_MULTIPLY, "*"},
+    {TOKEN_IDENT, "a"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+
+    {TOKEN_DISPLAY, "DISPLAY"},
+    {TOKEN_STRING, "\"The equation has two distinct real roots: \""},
+
+    {TOKEN_DISPLAY, "DISPLAY"},
+    {TOKEN_STRING, "\"Root 1: \""},
+    {TOKEN_IDENT, "root1"},
+
+    {TOKEN_DISPLAY, "DISPLAY"},
+    {TOKEN_STRING, "\"Root 2: \""},
+    {TOKEN_IDENT, "root2"},
+
+    // {TOKEN_EOF, ""},
+
+    {TOKEN_ELSE, "ELSE"},
+    {TOKEN_IF, "IF"},
+    {TOKEN_IDENT, "discriminant"},
+    {TOKEN_EQUAL, "="},
+    {TOKEN_INTEGER, "0"},
+
+    {TOKEN_KEYWORD_COMPUTE, "COMPUTE"},
+    {TOKEN_IDENT, "root1"},
+    {TOKEN_EQUAL, "="},
+    {TOKEN_SUB, "-"},
+    {TOKEN_IDENT, "b"},
+    {TOKEN_DIVIDE, "/"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_INTEGER, "2"},
+    {TOKEN_MULTIPLY, "*"},
+    {TOKEN_IDENT, "a"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+
+
+    {TOKEN_DISPLAY, "DISPLAY"},
+    {TOKEN_STRING, "\"The equation has one real root: \""},
+
+    {TOKEN_DISPLAY, "DISPLAY"},   
+    {TOKEN_STRING, "\"Root: \""},
+    {TOKEN_IDENT, "root1"},
+
+
+    {TOKEN_ELSE, "ELSE"},
+
+    {TOKEN_DISPLAY, "DISPLAY"},
+    {TOKEN_STRING, "\"The equation has no real roots.\""},
+    {TOKEN_END_IF, "END-IF"},
+
+    // {TOKEN_EOF, ""},
+
+    {TOKEN_STOP, "STOP"},
+    {TOKEN_RUN, "RUN"},
+    {TOKEN_DOT, "."},
+    {TOKEN_EOF, ""},
+  };
+
+  yyin = fopen("samples/quadratic-snippet.cbl", "r");
+  ASSERT_TRUE(yyin);
+  int index = 0;
+  yytoken_kind_t t;
+  do {
+    printf("index: %d token: %d text: %s\n", index, t, yytext);
+    ASSERT_EQ(tokens[index].t, (t = yylex()));
+    ASSERT_STREQ(tokens[index].p, yytext);
+    ++index;
+  } while (t != TOKEN_EOF);
+}
+
+UTEST(scanner, sorting) {
+  struct token_st tokens[] = {
+    {TOKEN_KEYWORD_IDENTIFICATION, "IDENTIFICATION"},
+    {TOKEN_KEYWORD_DIVISION, "DIVISION"},
+    {TOKEN_DOT, "."},
+    {TOKEN_PROGRAM_ID, "PROGRAM-ID"},
+    {TOKEN_DOT, "."},
+    {TOKEN_IDENT, "sorting"},
+    {TOKEN_DOT, "."},
+    {TOKEN_KEYWORD_DATA, "DATA"},
+    {TOKEN_KEYWORD_DIVISION, "DIVISION"},
+    {TOKEN_DOT, "."},
+    {TOKEN_WORKING_STORAGE, "WORKING-STORAGE"},
+    {TOKEN_KEYWORD_SECTION, "SECTION"},
+    {TOKEN_DOT, "."},
+    {TOKEN_INTEGER, "01"},
+    {TOKEN_IDENT, "WS-SORT-AREA"},
+    {TOKEN_DOT, "."},
+    {TOKEN_INTEGER, "05"},
+    {TOKEN_IDENT, "WS-SORT-TABLE"},
+    {TOKEN_DOT, "."},
+    {TOKEN_INTEGER, "10"},
+    {TOKEN_IDENT, "WS-SORT-ROW"},
+    {TOKEN_PICTURE, "PIC"},
+    {TOKEN_ALPHANUMERIC, "X"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_INTEGER, "10"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_KEYWORD_OCCURS, "OCCURS"},
+    {TOKEN_INTEGER, "100"},
+    {TOKEN_DOT, "."},
+    {TOKEN_INTEGER, "05"},
+    {TOKEN_IDENT, "WS-TEMP-ROW"},
+    {TOKEN_PICTURE, "PIC"},
+    {TOKEN_ALPHANUMERIC, "X"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_INTEGER, "10"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_DOT, "."},
+    {TOKEN_INTEGER, "05"},
+    {TOKEN_IDENT, "WS-ROW-MAX"},
+    {TOKEN_PICTURE, "PIC"},
+    {TOKEN_SIGNED_NUMERIC, "S9"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_INTEGER, "4"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_COMPUTATION_LEVEL_0, "COMP"},
+    {TOKEN_KEYWORD_VALUE, "VALUE"},
+    {TOKEN_INTEGER, "100"},
+    {TOKEN_DOT, "."},
+    {TOKEN_INTEGER, "05"},
+    {TOKEN_IDENT, "WS-SORT-MAX"},
+    {TOKEN_PICTURE, "PIC"},
+    {TOKEN_SIGNED_NUMERIC, "S9"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_INTEGER, "4"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_COMPUTATION_LEVEL_0, "COMP"},
+    {TOKEN_DOT, "."},
+    {TOKEN_INTEGER, "05"},
+    {TOKEN_IDENT, "WS-I"},
+    {TOKEN_PICTURE, "PIC"},
+    {TOKEN_SIGNED_NUMERIC, "S9"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_INTEGER, "4"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_COMPUTATION_LEVEL_0, "COMP"},
+    {TOKEN_DOT, "."},
+    {TOKEN_INTEGER, "05"},
+    {TOKEN_IDENT, "WS-J"},
+    {TOKEN_PICTURE, "PIC"},
+    {TOKEN_SIGNED_NUMERIC, "S9"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_INTEGER, "4"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_COMPUTATION_LEVEL_0, "COMP"},
+    {TOKEN_DOT, "."},
+    {TOKEN_INTEGER, "05"},
+    {TOKEN_IDENT, "WS-INDEX"},
+    {TOKEN_PICTURE, "PIC"},
+    {TOKEN_SIGNED_NUMERIC, "S9"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_INTEGER, "4"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_COMPUTATION_LEVEL_0, "COMP"},
+    {TOKEN_DOT, "."},
+    {TOKEN_PROCEDURE, "PROCEDURE"},
+    {TOKEN_KEYWORD_DIVISION, "DIVISION"},
+    {TOKEN_DOT, "."},
+    {TOKEN_MOVE, "MOVE"},
+    {TOKEN_STRING, "\"30\""},
+    {TOKEN_KEYWORD_TO, "TO"},
+    {TOKEN_IDENT, "WS-SORT-ROW"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_INTEGER, "1"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_MOVE, "MOVE"},
+    {TOKEN_STRING, "\"10\""},
+    {TOKEN_KEYWORD_TO, "TO"},
+    {TOKEN_IDENT, "WS-SORT-ROW"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_INTEGER, "2"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_MOVE, "MOVE"},
+    {TOKEN_STRING, "\"50\""},
+    {TOKEN_KEYWORD_TO, "TO"},
+    {TOKEN_IDENT, "WS-SORT-ROW"},
+    {TOKEN_LEFT_PARENTHESIS, "("},  
+    {TOKEN_INTEGER, "3"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_MOVE, "MOVE"},
+    {TOKEN_STRING, "\"20\""},
+    {TOKEN_KEYWORD_TO, "TO"},
+    {TOKEN_IDENT, "WS-SORT-ROW"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_INTEGER, "4"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_MOVE, "MOVE"},
+    {TOKEN_STRING, "\"40\""},
+    {TOKEN_KEYWORD_TO, "TO"},
+    {TOKEN_IDENT, "WS-SORT-ROW"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_INTEGER, "5"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_MOVE, "MOVE"},
+    {TOKEN_INTEGER, "5"},
+    {TOKEN_KEYWORD_TO, "TO"},
+    {TOKEN_IDENT, "WS-SORT-MAX"},
+    {TOKEN_DISPLAY, "DISPLAY"},
+    {TOKEN_STRING, "\"Original Array Contents:\""},
+    {TOKEN_DISPLAY, "DISPLAY"},
+    {TOKEN_STRING, "\"---------------------\""},
+    {TOKEN_PERFORM, "PERFORM"},
+    {TOKEN_VARYING, "VARYING"},
+    {TOKEN_IDENT, "WS-INDEX"},
+    {TOKEN_KEYWORD_FROM, "FROM"},
+    {TOKEN_INTEGER, "1"},
+    {TOKEN_KEYWORD_BY, "BY"},
+    {TOKEN_INTEGER, "1"},
+    {TOKEN_UNTIL, "UNTIL"},
+    {TOKEN_IDENT, "WS-INDEX"},
+    {TOKEN_GREATER_THAN, ">"},
+    {TOKEN_IDENT, "WS-SORT-MAX"},
+    {TOKEN_DISPLAY, "DISPLAY"},
+    {TOKEN_STRING, "\"Element \""},
+    {TOKEN_IDENT, "WS-INDEX"},
+    {TOKEN_STRING, "\": \""},
+    {TOKEN_IDENT, "WS-SORT-ROW"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_IDENT, "WS-INDEX"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_END_PERFORM, "END-PERFORM"},
+    {TOKEN_DISPLAY, "DISPLAY"},
+    {TOKEN_SPACE, "SPACE"},
+    {TOKEN_PERFORM, "PERFORM"},
+    {TOKEN_VARYING, "VARYING"},
+    {TOKEN_IDENT, "WS-I"},
+    {TOKEN_KEYWORD_FROM, "FROM"},
+    {TOKEN_INTEGER, "1"},
+    {TOKEN_KEYWORD_BY, "BY"},
+    {TOKEN_INTEGER, "1"},
+    {TOKEN_UNTIL, "UNTIL"},
+    {TOKEN_IDENT, "WS-I"},
+    {TOKEN_GREATER_THAN, ">"},
+    {TOKEN_IDENT, "WS-SORT-MAX"},
+    {TOKEN_SUB, "-"},
+    {TOKEN_INTEGER, "1"},
+    {TOKEN_PERFORM, "PERFORM"},
+    {TOKEN_VARYING, "VARYING"},
+    {TOKEN_IDENT, "WS-J"},
+    {TOKEN_KEYWORD_FROM, "FROM"},
+    {TOKEN_INTEGER, "1"},
+    {TOKEN_KEYWORD_BY, "BY"},
+    {TOKEN_INTEGER, "1"},
+    {TOKEN_UNTIL, "UNTIL"},
+    {TOKEN_IDENT, "WS-J"},
+    {TOKEN_GREATER_THAN, ">"},
+    {TOKEN_IDENT, "WS-SORT-MAX"},
+    {TOKEN_SUB, "-"},
+    {TOKEN_IDENT, "WS-I"},
+    {TOKEN_IF, "IF"},
+    {TOKEN_IDENT, "WS-SORT-ROW"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_IDENT, "WS-J"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_GREATER_THAN, ">"},
+    {TOKEN_IDENT, "WS-SORT-ROW"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_IDENT, "WS-J"},
+    {TOKEN_ADD, "+"},
+    {TOKEN_INTEGER, "1"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_MOVE, "MOVE"},
+    {TOKEN_IDENT, "WS-SORT-ROW"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_IDENT, "WS-J"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_KEYWORD_TO, "TO"},
+    {TOKEN_IDENT, "WS-TEMP-ROW"},
+    {TOKEN_MOVE, "MOVE"},
+    {TOKEN_IDENT, "WS-SORT-ROW"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_IDENT, "WS-J"},
+    {TOKEN_ADD, "+"},
+    {TOKEN_INTEGER, "1"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_KEYWORD_TO, "TO"},
+    {TOKEN_IDENT, "WS-SORT-ROW"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_IDENT, "WS-J"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_MOVE, "MOVE"},
+    {TOKEN_IDENT, "WS-TEMP-ROW"},
+    {TOKEN_KEYWORD_TO, "TO"},
+    {TOKEN_IDENT, "WS-SORT-ROW"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_IDENT, "WS-J"},
+    {TOKEN_ADD, "+"},
+    {TOKEN_INTEGER, "1"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_END_IF, "END-IF"},
+    {TOKEN_END_PERFORM, "END-PERFORM"},
+    {TOKEN_END_PERFORM, "END-PERFORM"},
+    {TOKEN_DISPLAY, "DISPLAY"},
+    {TOKEN_STRING, "\"Sorted Array Contents:\""},
+    {TOKEN_DISPLAY, "DISPLAY"},
+    {TOKEN_STRING, "\"--------------------\""},
+    {TOKEN_PERFORM, "PERFORM"},
+    {TOKEN_VARYING, "VARYING"},
+    {TOKEN_IDENT, "WS-INDEX"},
+    {TOKEN_KEYWORD_FROM, "FROM"},
+    {TOKEN_INTEGER, "1"},
+    {TOKEN_KEYWORD_BY, "BY"},
+    {TOKEN_INTEGER, "1"},
+    {TOKEN_UNTIL, "UNTIL"},
+    {TOKEN_IDENT, "WS-INDEX"},
+    {TOKEN_GREATER_THAN, ">"},
+    {TOKEN_IDENT, "WS-SORT-MAX"},
+    {TOKEN_DISPLAY, "DISPLAY"},
+    {TOKEN_STRING, "\"Element \""},
+    {TOKEN_IDENT, "WS-INDEX"},
+    {TOKEN_STRING, "\": \""},
+    {TOKEN_IDENT, "WS-SORT-ROW"},
+    {TOKEN_LEFT_PARENTHESIS, "("},
+    {TOKEN_IDENT, "WS-INDEX"},
+    {TOKEN_RIGHT_PARENTHESIS, ")"},
+    {TOKEN_END_PERFORM, "END-PERFORM"},
+    {TOKEN_DOT, "."},
+    {TOKEN_STOP, "STOP"},
+    {TOKEN_RUN, "RUN"},
+    {TOKEN_DOT, "."},
+    {TOKEN_EOF, ""},
+  };
+  yyin = fopen("samples/sorting-snippet.cbl", "r");
+  ASSERT_TRUE(yyin);
+  int index = 0;
+  yytoken_kind_t t;
+  do {
+    printf("index: %d token: %d text: %s\n", index, t, yytext);
+    ASSERT_EQ(tokens[index].t, (t = yylex()));
+    ASSERT_STREQ(tokens[index].p, yytext);
+    ++index;
+  } while (t != TOKEN_EOF);
+}