Add support for "other" in preprocessing-tokens.

1 files changed, 14 insertions, 2 deletions

diff --git a/module/c/lex2.scm b/module/c/lex2.scm
index fcddcdc4..72f79f55 100644
--- a/module/c/lex2.scm
+++ b/module/c/lex2.scm
@@ -325,15 +325,25 @@
 (define-peg-pattern comment all
   (or line-comment block-comment))
 
+(define-peg-pattern non-whitespace all
+  (and (not-followed-by whitespace)
+       peg-any))
+
 (define-peg-pattern preprocessing-tokens all
   (* (or whitespace
          comment
-         preprocessing-token)))
+         preprocessing-token
+         non-whitespace)))
 
 
 
+;; comment could be merged with whitespace, but then unlex would have to know that
+
+;; other is the "each non-white-space character that cannot be one of the above"
+;; clause from 6.4 p. 1
+
 (define-type (lexeme)
-  (type type: (memv '(whitespace comment preprocessing-token placemaker)))
+  (type type: (memv '(whitespace comment preprocessing-token other placemaker)))
   (body type: (or string? list?))
   (noexpand type: (list-of string?)
             default: '()))
@@ -343,6 +353,8 @@
 
 (define (lex-output->lexeme-object x)
   (match x
+    (`(non-whitespace ,body)
+     (lexeme body: body type: 'other))
     (`(whitespace ,body)
      (lexeme body: body type: 'whitespace ))
     (`(comment ,body)