1 files changed, 65 insertions, 0 deletions
diff --git a/tests/test/cpp/to-token.scm b/tests/test/cpp/to-token.scm
new file mode 100644
index 00000000..b633ce12
--- /dev/null
+++ b/tests/test/cpp/to-token.scm
@@ -0,0 +1,65 @@
+(define-module (test cpp to-token)
+  :use-module (srfi srfi-64)
+  :use-module (srfi srfi-64 test-error)
+  :use-module (srfi srfi-88)
+  :use-module (c to-token)
+  :use-module ((system base lalr)
+               :select (lexical-token-category
+                        lexical-token-value))
+  :use-module ((c lex2) :select (lex))
+  )
+
+(test-group "string tokens"
+  (let ((v (preprocessing-token->token (car (lex "\"Hello\"")))))
+    (test-equal 'string-literal (lexical-token-category v))
+    (test-equal #vu8(#x48 #x65 #x6C #x6C #x6F 0) (lexical-token-value v))
+    ;; TODO prefixes
+    ))
+
+(test-group "identifier tokens"
+  (let ((v (preprocessing-token->token (car (lex "hello")))))
+    (test-equal 'identifier (lexical-token-category v))
+    (test-equal 'hello (lexical-token-value v))))
+
+(test-group "keywords"
+  (test-equal 'auto (preprocessing-token->token (car (lex "auto")))))
+
+(test-group "numbers"
+  (test-group "Integers"
+    (test-group "Base-10"
+      (let ((v (preprocessing-token->token (car (lex "1")))))
+        (test-equal 'constant (lexical-token-category v))
+        (test-equal 1 (lexical-token-value v))))
+
+    (test-equal "Base-16"
+      16 (lexical-token-value (preprocessing-token->token (car (lex "0x10")))))
+    (test-equal "Base-8"
+      8 (lexical-token-value (preprocessing-token->token (car (lex "010")))))
+    (test-group "Suffixes"
+      'TODO
+      ))
+
+  ;; TODO floats
+  )
+
+(test-group "character constants"
+  (let ((v (preprocessing-token->token (car (lex "'a'")))))
+    (test-equal 'constant (lexical-token-category v))
+    (test-equal (char->integer #\a) (lexical-token-value v)) )
+  (let ((v (preprocessing-token->token (car (lex "'ab'")))))
+    (test-equal 'constant (lexical-token-category v))
+    (test-equal (char->integer #\b) (lexical-token-value v)))
+  (let ((v (preprocessing-token->token (car (lex "'\\x41'")))))
+    (test-equal 'constant (lexical-token-category v))
+    (test-equal #x41 (lexical-token-value v)))
+  ;; (lex "'\\x4142'")
+  ;; (lex "'L\\x4142'")
+  )
+
+(test-group "punctuators"
+  (test-equal '+ (preprocessing-token->token (car (lex "+"))))
+  (test-equal 'lbrace (preprocessing-token->token (car (lex "{")))))
+
+(test-group "other"
+  (test-error 'cpp-error (preprocessing-token->token (car (lex " ")))))
+