## cr-tknzr cr-tknzr |

## Synopsis

    struct              CRTknzr;
                        CRTknzrPriv;
    CRTknzr *           cr_tknzr_new                        (CRInput *a_input);
    CRTknzr *           cr_tknzr_new_from_uri               (const guchar *a_file_uri,
                                                             enum CREncoding a_enc);
    CRTknzr *           cr_tknzr_new_from_buf               (guchar *a_buf,
                                                             gulong a_len,
                                                             enum CREncoding a_enc,
                                                             gboolean a_free_at_destroy);
    gboolean            cr_tknzr_unref                      (CRTknzr *a_this);
    void                cr_tknzr_ref                        (CRTknzr *a_this);
    enum CRStatus       cr_tknzr_read_byte                  (CRTknzr *a_this,
                                                             guchar *a_byte);
    enum CRStatus       cr_tknzr_read_char                  (CRTknzr *a_this,
                                                             guint32 *a_char);
    enum CRStatus       cr_tknzr_peek_char                  (CRTknzr *a_this,
                                                             guint32 *a_char);
    enum CRStatus       cr_tknzr_peek_byte                  (CRTknzr *a_this,
                                                             gulong a_offset,
                                                             guchar *a_byte);
    guchar              cr_tknzr_peek_byte2                 (CRTknzr *a_this,
                                                             gulong a_offset,
                                                             gboolean *a_eof);
    enum CRStatus       cr_tknzr_set_cur_pos                (CRTknzr *a_this,
                                                             CRInputPos *a_pos);
    glong               cr_tknzr_get_nb_bytes_left          (CRTknzr *a_this);
    enum CRStatus       cr_tknzr_get_cur_pos                (CRTknzr *a_this,
                                                             CRInputPos *a_pos);
    enum CRStatus       cr_tknzr_get_parsing_location       (CRTknzr *a_this,
                                                             CRParsingLocation *a_loc);
    enum CRStatus       cr_tknzr_seek_index                 (CRTknzr *a_this,
                                                             enum CRSeekPos a_origin,
                                                             gint a_pos);
    enum CRStatus       cr_tknzr_get_cur_byte_addr          (CRTknzr *a_this,
                                                             guchar **a_addr);
    enum CRStatus       cr_tknzr_consume_chars              (CRTknzr *a_this,
                                                             guint32 a_char,
                                                             glong *a_nb_char);
    enum CRStatus       cr_tknzr_get_next_token             (CRTknzr *a_this,
                                                             CRToken **a_tk);
    enum CRStatus       cr_tknzr_unget_token                (CRTknzr *a_this,
                                                             CRToken *a_token);
    enum CRStatus       cr_tknzr_parse_token                (CRTknzr *a_this,
                                                             enum CRTokenType a_type,
                                                             enum CRTokenExtraType a_et,
                                                             gpointer a_res,
                                                             gpointer a_extra_res);
    enum CRStatus       cr_tknzr_set_input                  (CRTknzr *a_this,
                                                             CRInput *a_input);
    enum CRStatus       cr_tknzr_get_input                  (CRTknzr *a_this,
                                                             CRInput **a_input);
    void                cr_tknzr_destroy                    (CRTknzr *a_this);

## Description

## Details

### struct CRTknzr

    struct CRTknzr {
            /*the private data of the tokenizer.*/
            CRTknzrPriv *priv ;
    };

### CRTknzrPriv

    typedef struct _CRTknzrPriv CRTknzrPriv;

### cr_tknzr_new ()

    CRTknzr *           cr_tknzr_new                        (CRInput *a_input);

### cr_tknzr_new_from_uri ()

    CRTknzr *           cr_tknzr_new_from_uri               (const guchar *a_file_uri,
                                                             enum CREncoding a_enc);

### cr_tknzr_new_from_buf ()

    CRTknzr *           cr_tknzr_new_from_buf               (guchar *a_buf,
                                                             gulong a_len,
                                                             enum CREncoding a_enc,
                                                             gboolean a_free_at_destroy);

### cr_tknzr_unref ()

    gboolean            cr_tknzr_unref                      (CRTknzr *a_this);

### cr_tknzr_ref ()

    void                cr_tknzr_ref                        (CRTknzr *a_this);

### cr_tknzr_read_byte ()

    enum CRStatus       cr_tknzr_read_byte                  (CRTknzr *a_this,
                                                             guchar *a_byte);

### cr_tknzr_read_char ()

    enum CRStatus       cr_tknzr_read_char                  (CRTknzr *a_this,
                                                             guint32 *a_char);

### cr_tknzr_peek_char ()

    enum CRStatus       cr_tknzr_peek_char                  (CRTknzr *a_this,
                                                             guint32 *a_char);

### cr_tknzr_peek_byte ()

    enum CRStatus       cr_tknzr_peek_byte                  (CRTknzr *a_this,
                                                             gulong a_offset,
                                                             guchar *a_byte);

### cr_tknzr_peek_byte2 ()

    guchar              cr_tknzr_peek_byte2                 (CRTknzr *a_this,
                                                             gulong a_offset,
                                                             gboolean *a_eof);

### cr_tknzr_set_cur_pos ()

    enum CRStatus       cr_tknzr_set_cur_pos                (CRTknzr *a_this,
                                                             CRInputPos *a_pos);

### cr_tknzr_get_nb_bytes_left ()

    glong               cr_tknzr_get_nb_bytes_left          (CRTknzr *a_this);

### cr_tknzr_get_cur_pos ()

    enum CRStatus       cr_tknzr_get_cur_pos                (CRTknzr *a_this,
                                                             CRInputPos *a_pos);

### cr_tknzr_get_parsing_location ()

    enum CRStatus       cr_tknzr_get_parsing_location       (CRTknzr *a_this,
                                                             CRParsingLocation *a_loc);

### cr_tknzr_seek_index ()

    enum CRStatus       cr_tknzr_seek_index                 (CRTknzr *a_this,
                                                             enum CRSeekPos a_origin,
                                                             gint a_pos);

### cr_tknzr_get_cur_byte_addr ()

    enum CRStatus       cr_tknzr_get_cur_byte_addr          (CRTknzr *a_this,
                                                             guchar **a_addr);

### cr_tknzr_consume_chars ()

    enum CRStatus       cr_tknzr_consume_chars              (CRTknzr *a_this,
                                                             guint32 a_char,
                                                             glong *a_nb_char);

### cr_tknzr_get_next_token ()

    enum CRStatus       cr_tknzr_get_next_token             (CRTknzr *a_this,
                                                             CRToken **a_tk);

### cr_tknzr_unget_token ()

    enum CRStatus       cr_tknzr_unget_token                (CRTknzr *a_this,
                                                             CRToken *a_token);

### cr_tknzr_parse_token ()

    enum CRStatus       cr_tknzr_parse_token                (CRTknzr *a_this,
                                                             enum CRTokenType a_type,
                                                             enum CRTokenExtraType a_et,
                                                             gpointer a_res,
                                                             gpointer a_extra_res);

### cr_tknzr_set_input ()

    enum CRStatus       cr_tknzr_set_input                  (CRTknzr *a_this,
                                                             CRInput *a_input);

### cr_tknzr_get_input ()

    enum CRStatus       cr_tknzr_get_input                  (CRTknzr *a_this,
                                                             CRInput **a_input);

### cr_tknzr_destroy ()

    void                cr_tknzr_destroy                    (CRTknzr *a_this);