thodg/css-lexer

diff --git a/css-lexer.asd b/css-lexer.asd
index 64351b7..073f452 100644
--- a/css-lexer.asd
+++ b/css-lexer.asd
@@ -28,7 +28,13 @@
   :author "Thomas de Grivel <thoxdg@gmail.com>"
   :version "0.1"
   :description "CSS lexer as a cl-stream/token-stream"
-  :depends-on ("cl-stream")
+  :depends-on ("token-stream")
   :components
   ((:file "package")
    (:file "css-lexer" :depends-on ("package"))))
+
+(defsystem :css-lexer/test
+  :name "css-lexer/test"
+  :depends-on ("babel-stream" "css-lexer" "unistd-stream")
+  :components
+  ((:file "test")))
diff --git a/css-lexer.lisp b/css-lexer.lisp
index 2c99ae2..40b667a 100644
--- a/css-lexer.lisp
+++ b/css-lexer.lisp
@@ -1,6 +1,10 @@
 
 (in-package :css-lexer)
 
+(defclass css-lexer (lexer)
+  ((eof-p :initform nil
+          :accessor eof-p)))
+
 (defgeneric match-comment (lexer))
 (defgeneric match-newline (lexer))
 (defgeneric match-whitespace (lexer))
@@ -34,6 +38,7 @@
 (defgeneric column-token (lexer))
 (defgeneric cdo-token (lexer))
 (defgeneric cdc-token (lexer))
+(defgeneric consume-token (lexer))
 
 (defclass css-token (token) ())
 
@@ -251,7 +256,7 @@
 			      (match-url-unquoted lx))))
 		 (match-ws* lx)
 		 (when (match lx #\))
-		   (make-token lx (url-token :ident ident :url url)))))))
+		   (make-token lx 'url-token :ident ident :url url))))))
       (discard-token lx)))
 
 (defmethod match-digit ((lx lexer))
@@ -436,9 +441,13 @@
 
 (defmethod eof-token ((lx lexer))
   (push-token lx)
-  (if (lexer-eof lx)
-      (make-token lx 'eof-token)
-      (discard-token lx)))
+  (cond ((and (lexer-in-eof lx)
+              (= (lexer-match-start lx)
+                 (fill-pointer (lexer-buffer lx))))
+         (setf (eof-p lx) t)
+         (make-token lx 'eof-token))
+        (t
+         (discard-token lx))))
 
 (defmethod delim-token ((lx lexer))
   (push-token lx)
@@ -448,14 +457,12 @@
 
 ;;  CSS lexer
 
-(defclass css-lexer (lexer)
-  ())
-
 (defmethod stream-element-type ((lx css-lexer))
   'css-token)
 
-(defmethod stream-read ((lx css-lexer))
-  (or (whitespace-token lx)
+(defmethod lexer-token ((lx css-lexer))
+  (or (eof-token lx)
+      (whitespace-token lx)
       (string-token lx)
       (hash-token lx)
       (suffix-match-token lx)
@@ -479,8 +486,8 @@
       (ident-token lx)
       (dash-match-token lx)
       (include-match-token lx)
-      (eof-token lx)
-      (delim-token lx)))
+      (delim-token lx)
+      (error "no matching css token")))
 
 (defun css-lexer (stream)
   (assert (eq 'character (stream-element-type stream)))
diff --git a/test.lisp b/test.lisp
index 18ceda2..2efb40d 100644
--- a/test.lisp
+++ b/test.lisp
@@ -1,26 +1,32 @@
 
 (in-package :common-lisp-user)
 
-(require :babel-stream)
-(require :unistd-stream)
-(require :token-stream)
-
-(defpackage :css-lexer-tests
+(defpackage :css-lexer/test
   (:use :babel-stream
         :cl
         :cl-stream
         :css-lexer
         :unistd-stream)
   #.(cl-stream:shadowing-import-from)
-  (:export #:run))
+  (:export
+   #:run
+   #:simple-test))
+
+(in-package :css-lexer/test)
+
+(defun simple-test ()
+  (with-stream (css (css-lexer
+                     (string-input-stream
+                      "body { color: #f00; }")))
+    (loop
+       (let ((r (stream-read css)))
+         (print r)
+         (when (typep r 'css-lexer::eof-token)
+           (return))))))
 
-(in-package :css-lexer-tests)
+(defun run ()
+  (asdf:load-system :css-lexer/test)
+  (simple-test))
 
-(with-stream (css (css-lexer
-                   (string-input-stream
-                    "body { color: #f00; }")))
-  (loop
-     (let ((r (stream-read css)))
-       (when (typep r 'css-lexer::eof-token)
-         (return))
-       (print r))))
+(untrace token-stream::subseq*
+         token-stream:make-token)