Add support for tokenizing a pre-allocated string instead of a file.

Use the new string tokenization to handle some simple command
line switches like -Dxxxx=yyyy - by simply adding lines to a
buffer that is tokenized and inserted before the actual file.

1 files changed, 3 insertions, 1 deletions

diff --git a/token.h b/token.h
index a5f24e0..7c8ec69 100644
--- a/token.h
+++ b/token.h
@@ -146,7 +146,9 @@ extern struct token *built_in_token(int, const char *);
 extern const char *show_special(int);
 extern const char *show_ident(const struct ident *);
 extern const char *show_token(const struct token *);
-extern struct token * tokenize(const char *, int, struct token *end);
+extern struct token * tokenize(const char *, int, struct token *);
+extern struct token * tokenize_buffer(unsigned char *, unsigned long, struct token *);
+
 extern void die(const char *, ...);
 extern void show_identifier_stats(void);
 extern struct token *preprocess(struct token *);