1

#ifndef LIB_LEXER_H

     2

#define LIB_LEXER_H

     3

     4

/*

     5

 * Simple FSM lexing

     6

 *

     7

 * The lexer is implemented as a Finite State Machine, consisting for a number of states, which then contain a set of

     8

 * transitions, which move the lexer from state to state based on each char of input at a time.

     9

 *

    10

 * Whenever the state changes, the token callback is triggered with the collected token data.

    11

 */

    12

    13

/*

    22

 * A transition from one state to another.

    23

 */

    24

struct lex_transition {

    25

    // applies to chars [left, right]

    26

    char left, right;

    27

    31

    // next state to enter

    32

    int next_state;

    33

};

    34

    35

/*

    43

 * A state

    44

 */

    45

struct lex_state {

    46

    // the state name (for debugging)

    47

    const char *name;

    48

    52

    // list of transitions for this state, terminated by a transition with next_state=0

    53

    struct lex_transition *trans_list;

    54

};

    55

    56

/*

    57

 * Lex machine

    58

 */

    59

struct lex {

    60

    // number of states

    61

    size_t state_count;

    62

    63

    // array of lex_states, indexable by the state id.

    64

    struct lex_state *state_list;

    65

    66

    /*

    67

     * Core token handler. Everytime a full token is lexed (i.e. the state changes), this will be called.

    68

     * `this_token` represents the full token that was parsed, and `token_data` is the token's value. `next_token`

    69

     * is the state that terminated this token, and `prev_token` was the token before this one.

    70

     *

    71

     * `token_data` is a buffer allocated by the lexer that the actual input data is copied into. Thence, it can be

    72

     * modified, as its contents will be replaced by the next token. Hence, if you need to keep hold of it, copy it.

    73

     *

    74

     * Return zero to have lexing continue, nonzero to stop lexing.

    75

     */

    77

    78

    /*

    79

     * Called on every char handled by the lexer. `this_token` is the state of the token that the char belongs to.

    80

     *

    81

     * Return zero to have lexing continue, nonzero to stop lexing.

    82

     */

    84

    85

    /*

    86

     * Called when the end of input has been reached, `last_token` is the state that we terminated in.

    87

     *

    88

     * Return zero to indiciate that the input was valid, nonzero to indicate an error.

    89

     */

    91

};

    92

    93

/*

    94

 * Helper macros for building the state_list

    95

 */

    98

   101

    #define LEX_ALPHA(to)           LEX_RANGE('a', 'z', to), LEX_RANGE('A', 'Z', to)

   102

    #define LEX_NUMBER(to)          LEX_RANGE('0', '9', to)

   103

    #define LEX_ALNUM(to)           LEX_ALPHA(to), LEX_NUMBER(to), LEX_CHAR('-', to), LEX_CHAR('_', to)

   104

    #define LEX_WHITESPACE(to)      LEX_CHAR(' ', to), LEX_CHAR('\n', to), LEX_CHAR('\t', to)

   105

   110

   111

/*

   112

 * Lex it!

   113

 *

   114

 * Return zero to indiciate that the input was valid, nonzero otherwise.

   115

 */

   117

   118

#endif /* LIB_LEXER_H */