thodg/css-lexer/css-lexer.lisp

Branch - branch -master

• Show log

  Commit

• Hash : 60005ba9
  Author : Thomas de Grivel
  Date : 2022-11-04T11:09:33
  

  ws
  

Download

• css-lexer.lisp

• ;; css-lexer - CSS lexer as a cl-stream/token-stream
  ;; Copyright 2018,2022 Thomas de Grivel <thoxdg@gmail.com>
  
  (in-package :css-lexer)
  
  (defclass css-lexer (lexer) ())
  
  (defgeneric comment-token (css-lexer))
  (defgeneric match-newline (css-lexer))
  (defgeneric match-whitespace (css-lexer))
  (defgeneric match-hex-digit (css-lexer))
  (defgeneric match-escape (css-lexer))
  (defgeneric whitespace-token (css-lexer))
  (defgeneric match-ws* (css-lexer))
  (defgeneric match-ident-char (css-lexer))
  (defgeneric match-ident-char* (css-lexer))
  (defgeneric ident-token (css-lexer))
  (defgeneric function-token (css-lexer))
  (defgeneric at-keyword-token (css-lexer))
  (defgeneric match-string-char (css-lexer end-char))
  (defgeneric match-string (css-lexer end-char))
  (defgeneric string-token (css-lexer))
  (defgeneric match-non-printable (css-lexer))
  (defgeneric match-url-unquoted-char (css-lexer))
  (defgeneric match-url-unquoted (css-lexer))
  (defgeneric url-token (css-lexer))
  (defgeneric match-digit (css-lexer))
  (defgeneric match-digit+ (css-lexer))
  (defgeneric number-token (css-lexer))
  (defgeneric dimension-token (css-lexer))
  (defgeneric percentage-token (css-lexer))
  (defgeneric unicode-range-token (css-lexer))
  (defgeneric include-match-token (css-lexer))
  (defgeneric dash-match-token (css-lexer))
  (defgeneric prefix-match-token (css-lexer))
  (defgeneric suffix-match-token (css-lexer))
  (defgeneric substring-match-token (css-lexer))
  (defgeneric column-token (css-lexer))
  (defgeneric cdo-token (css-lexer))
  (defgeneric cdc-token (css-lexer))
  (defgeneric consume-token (css-lexer))
  
  (defclass css-token (token) ())
  (defclass printable (token) ())
  
  (defclass identified-token (css-token)
    ((ident :initarg :ident
  	  :reader token-ident
  	  :type ident-token)))
  
  (defclass comment-token (css-token) ())
  (defclass whitespace-token (css-token) ())
  (defclass ident-token (printable css-token) ())
  (defclass function-token (identified-token) ())
  (defclass at-keyword-token (identified-token) ())
  (defclass hash-token (printable css-token) ())
  (defclass string-token (css-token) ())
  
  (defclass url-token (identified-token)
    ((url :initarg :url
  	:reader token-url
  	:type token)))
  
  (defclass number-token (printable css-token) ())
  
  (defclass numbered-token (css-token)
    ((number :initarg :number
  	   :reader token-number
  	   :type number-token)))
  
  (defclass dimension-token (identified-token numbered-token) ())
  (defclass percentage-token (numbered-token) ())
  (defclass unicode-range-token (css-token) ())
  (defclass include-match-token (css-token) ())
  (defclass dash-match-token (css-token) ())
  (defclass prefix-match-token (css-token) ())
  (defclass suffix-match-token (css-token) ())
  (defclass substring-match-token (css-token) ())
  (defclass column-token (css-token) ())
  (defclass cdo-token (css-token) ())
  (defclass cdc-token (css-token) ())
  (defclass left-paren-token (css-token) ())
  (defclass right-paren-token (css-token) ())
  (defclass comma-token (css-token) ())
  (defclass colon-token (css-token) ())
  (defclass semicolon-token (css-token) ())
  (defclass [-token (css-token) ())
  (defclass ]-token (css-token) ())
  (defclass {-token (css-token) ())
  (defclass }-token (css-token) ())
  (defclass eof-token (css-token) ())
  (defclass delim-token (printable css-token) ())
  
  (defmethod comment-token ((lx css-lexer))
    (push-token lx)
    (or (and (match lx "/*")
             (match-until lx "*/")
             (make-token lx 'comment-token))
        (discard-token lx)))
  
  (let ((rn (coerce '(#\Return #\Newline) 'string)))
    (defmethod match-newline ((lx css-lexer))
      (or (match lx #\Newline)
          (match lx rn)
          (match lx #\Return)
          (match lx #\Linefeed))))
  
  (defmethod match-whitespace ((lx css-lexer))
    (or (match lx #\Space)
        (match lx #\Tab)
        (match-newline lx)))
  
  (defmethod match-hex-digit ((lx css-lexer))
    (let ((c (the fixnum (matcher-char lx 0))))
      (when (or (<= (char-code #\0) c (char-code #\9))
  	      (<= (char-code #\a) c (char-code #\f))
  	      (<= (char-code #\A) c (char-code #\F)))
        (incf (the fixnum (matcher-start lx))))))
         
  (defmethod match-escape ((lx css-lexer))
    (match-sequence lx
      (and (match lx #\\)
  	 (or (when (match-times lx #'match-hex-digit 1 6)
  	       (match-whitespace lx)
  	       (matcher-start lx))
  	     (when (not (match-newline lx))
  	       (matcher-start lx))))))
  
  (defmethod whitespace-token ((lx css-lexer))
    (push-token lx)
    (if (match-times lx #'match-whitespace 1 nil)
        (make-token lx 'whitespace-token)
        (discard-token lx)))
  
  (defmethod match-ws* ((lx css-lexer))
    (match-option lx #'whitespace-token))
  
  (defmethod match-ident-char ((lx css-lexer))
    (or (match-escape lx)
        (let ((c (the character (matcher-char lx 0))))
  	(when (or (char<= #\a c #\z)
  		  (char<= #\A c #\Z)
  		  (char<= #\0 c #\9)
  		  (char=  #\_ c)
  		  (char=  #\- c)
  		  (<  #x007F (char-code c)))
  	  (incf (the fixnum (matcher-start lx)))))))
  
  (defmethod match-ident-char* ((lx css-lexer))
    (match-times lx #'match-ident-char 0 nil))
  
  (defmethod ident-token ((lx css-lexer))
    (match-sequence lx
      (push-token lx)
      (match lx #\-)
      (cond ((or (match-escape lx)
  	       (let ((c (the character (matcher-char lx 0))))
  		 (when (or (char<= #\a c #\z)
  			   (char<= #\A c #\Z)
  			   (char=  #\_ c)
  			   (<  #x007F (char-code c)))
  		   (incf (the fixnum (matcher-start lx))))))
  	   (match-ident-char* lx)
  	   (make-token lx 'ident-token))
  	  (t
  	   (discard-token lx)))))
  
  (defmethod function-token ((lx css-lexer))
    (match-sequence lx
      (push-token lx)
      (let ((ident (ident-token lx)))
        (if (and ident (match lx #\())
  	  (make-token lx 'function-token :ident ident)
  	  (discard-token lx)))))
  
  (defmethod at-keyword-token ((lx css-lexer))
    (match-sequence lx
      (push-token lx)
      (if (match lx #\@)
  	(let ((ident (ident-token lx)))
  	  (if ident
  	      (make-token lx 'at-keyword-token :ident ident)
  	      (discard-token lx)))
  	(discard-token lx))))
  
  (defmethod hash-token ((lx css-lexer))
    (match-sequence lx
      (push-token lx)
      (if (match lx #\#)
  	(and (match-ident-char* lx)
  	     (make-token lx 'hash-token))
  	(discard-token lx))))
  
  (defgeneric string-token-string (string-token))
  
  (defmethod string-token-string ((s string-token))
    (let ((string (token-string s)))
      (declare (type (vector character) string))
      (subseq string 1 (1- (length string)))))
  
  (defmethod match-string-char ((lx css-lexer) (end-char character))
    (or (match-sequence lx
  	(and (match lx #\\)
  	     (match-newline lx)))
        (match-escape lx)
        (match-not lx (lambda (lx)
                        (or (match lx end-char)
                            (match lx #\\)
                            (match-newline lx))))))
  
  (defmethod match-string ((lx css-lexer) (end-char character))
    (match-sequence lx
      (and (match lx end-char)
           (match-times lx (lambda (lx) (match-string-char lx end-char)) 0 nil)
           (match lx end-char))))
  
  (defmethod string-token ((lx css-lexer))
    (push-token lx)
    (if (or (match-string lx #\")
  	  (match-string lx #\'))
        (make-token lx 'string-token)))
  
  (defmethod match-non-printable ((lx css-lexer))
    (let ((c (the fixnum (matcher-char lx 0))))
      (when (or (<= #x0000 c #x0008)
  	      (=  #x000B c)
  	      (<= #x000E c #x001F)
  	      (=  #x007F c))
        (incf (the fixnum (matcher-start lx))))))
  
  (defmethod match-url-unquoted-char ((lx css-lexer))
    (or (match-escape lx)
        (match-not lx
  	(or (match lx #\")
  	    (match lx #\')
  	    (match lx #\()
  	    (match lx #\))
  	    (match lx #\\)
  	    (match-whitespace lx)
  	    (match-non-printable lx)))))
  
  (defmethod match-url-unquoted ((lx css-lexer))
    (push-token lx)
    (if (match-times lx #'match-url-unquoted-char 1 nil)
        (make-token lx 'token)
        (discard-token lx)))
  
  (defmethod url-token ((lx css-lexer))
    (push-token lx)
    (or (match-sequence lx
  	(let ((ident (ident-token lx)))
  	  (and (string= "url" (the (vector character)
                                     (token-string ident)))
  	       (match lx #\()
  	       (match-ws* lx)
  	       (let ((url (or (string-token lx)
  			      (match-url-unquoted lx))))
  		 (match-ws* lx)
  		 (when (match lx #\))
  		   (make-token lx 'url-token :ident ident :url url))))))
        (discard-token lx)))
  
  (defmethod match-digit ((lx css-lexer))
    (let ((c (the character (matcher-char lx 0))))
      (when (char<= #\0 c #\9)
        (incf (the fixnum (matcher-start lx))))))
  
  (defmethod match-digit+ ((lx css-lexer))
    (match-times lx #'match-digit 1 nil))
  
  (defmethod number-token ((lx css-lexer))
    (push-token lx)
    (if (match-sequence lx
  	(and (or (match lx #\-)
  		 (match lx #\+)
  		 t)
  	     (or (match-sequence lx
  		   (and (match lx #\.)
  			(match-digit+ lx)))
  		 (match-sequence lx
  		   (match-digit+ lx)
  		   (match lx #\.)
  		   (match-digit+ lx))
  		 (match-digit+ lx))
  	     (or (match-sequence lx
  		   (and (or (match lx #\E)
  			    (match lx #\e))
  			(or (match lx #\-)
  			    (match lx #\+)
  			    t)
  			(match-digit+ lx)))
  		 (matcher-start lx))))
        (make-token lx 'number-token)
        (discard-token lx)))
  
  (defmethod dimension-token ((lx css-lexer))
    (push-token lx)
    (or (match-sequence lx
  	(let ((number (number-token lx)))
  	  (when number
  	    (let ((ident (ident-token lx)))
  	      (when ident
  		(make-token lx 'dimension-token
  			    :number number
  			    :ident ident))))))
        (discard-token lx)))
  
  (defmethod percentage-token ((lx css-lexer))
    (push-token lx)
    (or (match-sequence lx
  	(let ((number (number-token lx)))
  	  (when number
  	    (when (match lx #\%)
  	      (make-token lx 'percentage-token
  			  :number number)))))
        (discard-token lx)))
  
  (defmethod unicode-range-token ((lx css-lexer))
    (push-token lx)
    (or (match-sequence lx
  	(and (or (match lx #\u)
  		 (match lx #\U))
  	     (match lx #\+)
  	     (or (match-sequence lx
  		   (and (match-times lx #'match-hex-digit 1 6)
  			(match lx #\-)
  			(match-times lx #'match-hex-digit 1 6)))
  		 (match-sequence lx
  		   (let ((start (the fixnum (matcher-start lx))))
  		     (and (match-times lx #'match-hex-digit 0 5)
  			  (let ((digits (- (the fixnum
                                                  (matcher-start lx))
                                             start)))
                              (declare (type fixnum digits))
  			    (match-times lx (lambda (lx) (match lx #\?))
  					 1 (the fixnum (- 6 digits)))))))
  		 (match-times lx #'match-hex-digit 1 6))
  	     (make-token lx 'unicode-range-token)))
        (discard-token lx)))
  
  (defmethod include-match-token ((lx css-lexer))
    (push-token lx)
    (if (match lx "~=")
        (make-token lx 'include-match-token)
        (discard-token lx)))
  
  (defmethod dash-match-token ((lx css-lexer))
    (push-token lx)
    (if (match lx "|=")
        (make-token lx 'dash-match-token)
        (discard-token lx)))
  
  (defmethod prefix-match-token ((lx css-lexer))
    (push-token lx)
    (if (match lx "^=")
        (make-token lx 'prefix-match-token)
        (discard-token lx)))
  
  (defmethod suffix-match-token ((lx css-lexer))
    (push-token lx)
    (if (match lx "$=")
        (make-token lx 'suffix-match-token)
        (discard-token lx)))
  
  (defmethod substring-match-token ((lx css-lexer))
    (push-token lx)
    (if (match lx "*=")
        (make-token lx 'substring-match-token)
        (discard-token lx)))
  
  (defmethod column-token ((lx css-lexer))
    (push-token lx)
    (if (match lx "||")
        (make-token lx 'column-token)
        (discard-token lx)))
  
  (defmethod cdo-token ((lx css-lexer))
    (push-token lx)
    (if (match lx "<!--")
        (make-token lx 'cdo-token)
        (discard-token lx)))
  
  (defmethod cdc-token ((lx css-lexer))
    (push-token lx)
    (if (match lx "-->")
        (make-token lx 'cdc-token)
        (discard-token lx)))
  
  (defmethod left-paren-token ((lx css-lexer))
    (push-token lx)
    (if (match lx #\()
        (make-token lx 'left-paren-token)
        (discard-token lx)))
  
  (defmethod right-paren-token ((lx css-lexer))
    (push-token lx)
    (if (match lx #\))
        (make-token lx 'right-paren-token)
        (discard-token lx)))
  
  (defmethod comma-token ((lx css-lexer))
    (push-token lx)
    (if (match lx #\,)
        (make-token lx 'comma-token)
        (discard-token lx)))
  
  (defmethod colon-token ((lx css-lexer))
    (push-token lx)
    (if (match lx #\:)
        (make-token lx 'colon-token)
        (discard-token lx)))
  
  (defmethod semicolon-token ((lx css-lexer))
    (push-token lx)
    (if (match lx #\;)
        (make-token lx 'semicolon-token)
        (discard-token lx)))
  
  (defmethod [-token ((lx css-lexer))
    (push-token lx)
    (if (match lx #\[)
        (make-token lx '[-token)
        (discard-token lx)))
  
  (defmethod ]-token ((lx css-lexer))
    (push-token lx)
    (if (match lx #\])
        (make-token lx ']-token)
        (discard-token lx)))
  
  (defmethod {-token ((lx css-lexer))
    (push-token lx)
    (if (match lx #\{)
        (make-token lx '{-token)
        (discard-token lx)))
  
  (defmethod }-token ((lx css-lexer))
    (push-token lx)
    (if (match lx #\})
        (make-token lx '}-token)
        (discard-token lx)))
  
  (defmethod eof-token ((lx css-lexer))
    (unless (matcher-input-n lx 1)
      (push-token lx)
      (cond ((and (matcher-input-ended lx)
                  (= (matcher-start lx)
                     (fill-pointer (stream-input-buffer lx))))
             (setf (lexer-eof-p lx) t)
             (make-token lx 'eof-token))
            (t
             (discard-token lx)))))
  
  (defmethod delim-token ((lx css-lexer))
    (push-token lx)
    (matcher-input-n lx 1)
    (incf (the fixnum (matcher-start lx)))
    (make-token lx 'delim-token))
  
  ;;  CSS lexer
  
  (defmethod stream-element-type ((lx css-lexer))
    'css-token)
  
  (defmethod lexer-token ((lx css-lexer))
    (or (eof-token lx)
        (whitespace-token lx)
        (string-token lx)
        (hash-token lx)
        (suffix-match-token lx)
        (left-paren-token lx)
        (right-paren-token lx)
        (substring-match-token lx)
        (number-token lx)
        (comma-token lx)
        (cdc-token lx)
        (comment-token lx)
        (colon-token lx)
        (semicolon-token lx)
        (cdo-token lx)
        (at-keyword-token lx)
        ([-token lx)
        (]-token lx)
        (prefix-match-token lx)
        ({-token lx)
        (}-token lx)
        (unicode-range-token lx)
        (ident-token lx)
        (dash-match-token lx)
        (include-match-token lx)
        (delim-token lx)
        (error "no matching css token")))
  
  (defun css-lexer (stream)
    (assert (eq 'character (stream-element-type stream)))
    (make-instance 'css-lexer :stream stream))
  
  ;;  print-object
  
  (defmethod print-object ((object identified-token) stream)
    (declare (type cl:stream stream))
    (print-unreadable-object (object stream :type t)
      (format stream "~S" (token-ident object))))
  
  (defmethod print-object ((object numbered-token) stream)
    (declare (type cl:stream stream))
    (print-unreadable-object (object stream :type t)
      (format stream "~S" (token-number object))))
  
  (defmethod print-object ((object printable) stream)
    (declare (type cl:stream stream))
    (print-unreadable-object (object stream :type t)
      (format stream "~S" (token-string object))))
  
  (defmethod print-object ((object string-token) stream)
    (declare (type cl:stream stream))
    (print-unreadable-object (object stream :type t)
      (format stream "~S" (string-token-string object))))
  
  (defmethod print-object ((object token) stream)
    (declare (type cl:stream stream))
    (print-unreadable-object (object stream :type t)))
  

Download