Skip to content
main.lisp 74.4 KiB
Newer Older
wlott's avatar
wlott committed
;;; -*- Package: C; Log: C.Log -*-
;;;
;;; **********************************************************************
ram's avatar
ram committed
;;; This code was written as part of the CMU Common Lisp project at
;;; Carnegie Mellon University, and has been placed in the public domain.
;;;
(ext:file-comment "$Header: src/compiler/main.lisp $")
ram's avatar
ram committed
;;;
wlott's avatar
wlott committed
;;; **********************************************************************
;;;
;;;    This file contains the top-level interfaces to the compiler.
;;; 
;;; Written by Rob MacLachlan
;;;
ram's avatar
ram committed
(in-package "C")
(export '(*compile-progress* compile-from-stream *block-compile-default*
			     start-block end-block
			     *byte-compile-default*
			     *byte-compile-top-level*))
(in-package "LISP")
(export '(*compile-verbose* *compile-print* *compile-file-pathname*
ram's avatar
ram committed
			    *compile-file-truename*
			    compile-file-pathname))
wlott's avatar
wlott committed

pw's avatar
pw committed
(declaim (special *constants* *free-variables* *compile-component*
		  *code-vector* *next-location* *result-fixups*
		  *free-functions* *source-paths*
		  *continuation-number* *continuation-numbers*
		  *number-continuations* *tn-id* *tn-ids* *id-tns*
		  *label-ids* *label-id* *id-labels*
		  *undefined-warnings* *compiler-error-count*
		  *compiler-warning-count* *compiler-note-count*
		  *compiler-error-output* *compiler-error-bailout*
		  *compiler-trace-output*
		  *last-source-context* *last-original-source*
		  *last-source-form* *last-format-string* *last-format-args*
		  *last-message-count* *lexical-environment*
		  *coalesce-constants*))
wlott's avatar
wlott committed

(defvar *block-compile-default* :specified
  "The default value for the :Block-Compile argument to COMPILE-FILE.")
(declaim (type (member t nil :specified) *block-compile-default*))

;;; Exported:
(defvar *byte-compile-default* :maybe
  "The default value for the :Byte-Compile argument to COMPILE-FILE.")

;;; Exported:
(defvar *byte-compile-top-level* t
  "Similar to *BYTE-COMPILE-DEFAULT*, but controls the compilation of top-level
  forms (evaluated at load-time) when the :BYTE-COMPILE argument is :MAYBE
  (the default.)  When true, we decide to byte-compile.")
;;; Exported:
(defvar *loop-analyze* nil
  "Whether loop analysis should be done or not.")
;;; Value of the :byte-compile argument to the compiler.
(defvar *byte-compile* :maybe)

;;; Bound by COMPILE-COMPONENT to T when byte-compiling, and NIL when
ram's avatar
ram committed
;;; native compiling.  During IR1 conversion this can also be :MAYBE, in which
;;; case we must look at the policy, see (byte-compiling).
ram's avatar
ram committed
(defvar *byte-compiling* :maybe)
(declaim (type (member t nil :maybe) *byte-compile* *byte-compiling*
	       *byte-compile-default*))
(defvar compiler-version "1.1")
(pushnew :python *features*)
(setf (getf ext:*herald-items* :python)
      `("    Python " ,compiler-version ", target "
	,#'(lambda (stream)
	     (write-string (backend-version *backend*) stream))))
wlott's avatar
wlott committed

(defvar *check-consistency* nil)

(defvar *record-xref-info* nil
  "Whether the compiler should record cross-reference information.")
wlott's avatar
wlott committed
(defvar *all-components*)

;;; The current block compilation state.  These are initialized to the 
;;; :Block-Compile and :Entry-Points arguments that COMPILE-FILE was called
;;; with.  Subsequent START-BLOCK or END-BLOCK declarations alter the values.
wlott's avatar
wlott committed
;;;
;;; *Block-Compile-Argument* holds the original value of the :block-compile
;;; argument, which overrides any internal declarations.
;;;
wlott's avatar
wlott committed
(defvar *block-compile*)
(defvar *block-compile-argument*)
(declaim (type (member nil t :specified)
	       *block-compile* *block-compile-argument*))
(defvar *entry-points*)
(declaim (list *entry-points*))
wlott's avatar
wlott committed

wlott's avatar
wlott committed
;;; When block compiling, used by PROCESS-FORM to accumulate top-level lambdas
;;; resulting from compiling subforms.  (In reverse order.)
;;;
(defvar *top-level-lambdas*)
(declaim (list *top-level-lambdas*))
wlott's avatar
wlott committed

(defvar *compile-verbose* t
  "The default for the :VERBOSE argument to COMPILE-FILE.")
(defvar *compile-print* t
  "The default for the :PRINT argument to COMPILE-FILE.")
(defvar *compile-progress* nil
  "The default for the :PROGRESS argument to COMPILE-FILE.")

(defvar *compile-file-pathname* nil
  "The defaulted pathname of the file currently being compiled, or NIL if not
  compiling.")
(defvar *compile-file-truename* nil
  "The TRUENAME of the file currently being compiled, or NIL if not
  compiling.")

(declaim (type (or pathname null) *compile-file-pathname*
	       *compile-file-truename*))
wlott's avatar
wlott committed

;;; The values of *Package* and policy when compilation started.
wlott's avatar
wlott committed
;;;
(defvar *initial-package*)
(defvar *initial-cookie*)
(defvar *initial-interface-cookie*)
wlott's avatar
wlott committed

;;; The source-info structure for the current compilation.  This is null
;;; globally to indicate that we aren't currently in any identifiable
;;; compilation.
;;;
(defvar *source-info* nil)

(defvar *user-source-info* nil
  "The user supplied source-info for the current compilation.  
  This is the :source-info argument to COMPILE-FROM-STREAM and will be
  stored in the INFO slot of the DEBUG-SOURCE in code components and 
  in the user USER-INFO slot of STREAM-SOURCE-LOCATIONs.")
wlott's avatar
wlott committed

;;; Maybe-Mumble  --  Internal
;;;
;;;    Mumble conditional on *compile-progress*.
wlott's avatar
wlott committed
;;;
(defun maybe-mumble (&rest foo)
  (when *compile-progress*
wlott's avatar
wlott committed
    (apply #'compiler-mumble foo)))

wlott's avatar
wlott committed
(deftype object () '(or fasl-file core-object null))

(defvar *compile-object* nil)
(declaim (type object *compile-object*))


wlott's avatar
wlott committed

;;;; Component compilation:

(defparameter max-optimize-iterations 6
  "The upper limit on the number of times that we will consecutively do IR1
wlott's avatar
wlott committed
  optimization that doesn't introduce any new code.  A finite limit is
  necessary, since type inference may take arbitrarily long to converge.")

(defevent ir1-optimize-until-done "IR1-OPTIMIZE-UNTIL-DONE called.")
(defevent ir1-optimize-maxed-out "Hit MAX-OPTIMIZE-ITERATIONS limit.")

;;; IR1-Optimize-Until-Done  --  Internal
;;;
;;;    Repeatedly optimize Component until no further optimizations can be
;;; found or we hit our iteration limit.  When we hit the limit, we clear the
;;; component and block REOPTIMIZE flags to discourage the following
wlott's avatar
wlott committed
;;; optimization attempt from pounding on the same code.
;;;
(defun ir1-optimize-until-done (component)
  (declare (type component component))
  (maybe-mumble "Opt")
  (event ir1-optimize-until-done)
  (let ((count 0)
	(cleared-reanalyze nil))
    (loop
      (when (component-reanalyze component)
	(setf count 0)
	(setf cleared-reanalyze t)
wlott's avatar
wlott committed
	(setf (component-reanalyze component) nil))
      (setf (component-reoptimize component) nil)
      (ir1-optimize component)
      (cond ((component-reoptimize component)
	     (incf count)
	     (when (= count max-optimize-iterations)
	       (maybe-mumble "*")
	       (cond ((retry-delayed-transforms :optimize)
		      (maybe-mumble "+")
		      (setf count 0))
		     (t
		      (event ir1-optimize-maxed-out)
		      (setf (component-reoptimize component) nil)
		      (do-blocks (block component)
			(setf (block-reoptimize block) nil))
		      (return)))))
	    ((retry-delayed-transforms :optimize)
	     (setf count 0)
	     (maybe-mumble "+"))
	    (t
	     (return)))
wlott's avatar
wlott committed
      (maybe-mumble "."))
    (when cleared-reanalyze
      (setf (component-reanalyze component) t))
    (maybe-mumble " "))
wlott's avatar
wlott committed
  (undefined-value))

(defparameter *constraint-propagate* t)
(defparameter *reoptimize-after-type-check-max* 10)
  "*REOPTIMIZE-AFTER-TYPE-CHECK-MAX* exceeded.")
wlott's avatar
wlott committed


;;; DFO-AS-NEEDED  --  Internal
;;;
;;;    Iterate doing FIND-DFO until no new dead code is discovered.
;;;
(defun dfo-as-needed (component)
  (declare (type component component))
  (when (component-reanalyze component)
    (maybe-mumble "DFO")
    (loop
      (find-dfo component)
      (unless (component-reanalyze component)
	(maybe-mumble " ")
	(return))
      (maybe-mumble ".")))
  (undefined-value))


wlott's avatar
wlott committed
;;; IR1-Phases  --  Internal
;;;
;;;    Do all the IR1 phases for a non-top-level component.
;;;
(defun ir1-phases (component)
  (declare (type component component))
	(loop-count 1)
	(*delayed-transforms* nil))
    (declare (special *constraint-number* *delayed-transforms*))
     (ir1-optimize-until-done component)
     (when (or (component-new-functions component)
	       (component-reanalyze-functions component))
       (maybe-mumble "Locall ")
       (local-call-analyze component))
     (dfo-as-needed component)
     (when *constraint-propagate*
       (maybe-mumble "Constraint ")
       (constraint-propagate component))
     (when (retry-delayed-transforms :constraint)
       (maybe-mumble "Rtran "))
     ;; Delay the generation of type checks until the type constraints have
     ;; had time to propagate, else the compiler can confuse itself.
     (unless (and (or (component-reoptimize component)
		      (component-reanalyze component)
		      (component-new-functions component)
		      (component-reanalyze-functions component))
		  (< loop-count (- *reoptimize-after-type-check-max* 4)))
       (maybe-mumble "Type ")
       (generate-type-checks component)
       (unless (or (component-reoptimize component)
		   (component-reanalyze component)
		   (component-new-functions component)
		   (component-reanalyze-functions component))
	 (return)))
     (when (>= loop-count *reoptimize-after-type-check-max*)
       (maybe-mumble "[Reoptimize Limit]")
       (event reoptimize-maxed-out)
       (return))
     (incf loop-count)))
wlott's avatar
wlott committed


;;; Native-Compile-Component  --  Internal
wlott's avatar
wlott committed
;;;
(defun native-compile-component (component)
  (let ((*code-segment* nil)
	(*elsewhere* nil)
	(*elsewhere-label* nil))
wlott's avatar
wlott committed
    (maybe-mumble "GTN ")
    (gtn-analyze component)
    (maybe-mumble "LTN ")
    (ltn-analyze component)
ram's avatar
ram committed
    (maybe-mumble "Control ")
    (control-analyze component #'make-ir2-block)
wlott's avatar
wlott committed

    (when (ir2-component-values-receivers (component-info component))
      (maybe-mumble "Stack ")
      (stack-analyze component)
      ;;
      ;; Assign BLOCK-NUMBER for any cleanup blocks introduced by stack
      ;; analysis.  There shouldn't be any unreachable code after control, so
      ;; this won't delete anything.
      (dfo-as-needed component))
wlott's avatar
wlott committed

    (unwind-protect
	(progn
	  (maybe-mumble "IR2Tran ")
	  (init-assembler)
	  (entry-analyze component)
	  (ir2-convert component)
	  
	  (when (policy nil (>= speed cspeed))
	    (maybe-mumble "Copy ")
	    (copy-propagate component))
	  
	  (select-representations component)
	  
	  (when *check-consistency*
	    (maybe-mumble "Check2 ")
	    (check-ir2-consistency component))
	  
	  (delete-unreferenced-tns component)
	  
	  (maybe-mumble "Life ")
	  (lifetime-analyze component)
	  
	  (when *compile-progress*
	    (compiler-mumble "") ; Sync before doing random output.
	    (pre-pack-tn-stats component *compiler-error-output*))
	  
	  (when *check-consistency*
	    (maybe-mumble "CheckL ")
	    (check-life-consistency component))
	  (maybe-mumble "Pack ")
	  (pack component)
	  
	  (when *check-consistency*
	    (maybe-mumble "CheckP ")
	    (check-pack-consistency component))
	  
	  (when *compiler-trace-output*
	    (describe-component component *compiler-trace-output*)
	    (describe-ir2-component component *compiler-trace-output*))
	  
	  (maybe-mumble "Code ")
	  (multiple-value-bind
	      (length trace-table fixups)
	      (generate-code component)
	    (when (and *compiler-trace-output*
		       (backend-disassem-params *backend*))
		      (intl:gettext "~|~%Disassembly of code for ~S~2%") component)
	      (disassem:disassemble-assem-segment *code-segment*
						  *compiler-trace-output*
						  *backend*))
	    (etypecase *compile-object*
	      (fasl-file
	       (maybe-mumble "FASL")
	       (fasl-dump-component component *code-segment*
				    length trace-table fixups
				    *compile-object*))
	      (core-object
	       (maybe-mumble "Core")
	       (make-core-component component *code-segment*
				    length trace-table fixups
				    *compile-object*))
      (when *code-segment*
	(new-assem:release-segment *code-segment*))
      (when *elsewhere*
	(new-assem:release-segment *elsewhere*))))
  ;; We are done, so don't bother keeping anything around.
  (nuke-ir2-component component)
  (setf (component-info component) nil)
ram's avatar
ram committed
;;; BYTE-COMPILING  --  Interface
;;;
;;;    Return our best guess for whether we will byte compile code currently
;;; being IR1 converted.  Only a guess because the decision is made on a
ram's avatar
ram committed
;;;
ram's avatar
ram committed
  (if (eq *byte-compiling* :maybe)
	  (policy nil (zerop speed) (<= debug 1)))
      (and *byte-compile* *byte-compiling*)))

;;; DELETE-IF-NO-ENTRIES  --  Internal
;;;
;;;    Delete components with no external entry points before we try to
;;; generate code.  Unreachable closures can cause IR2 conversion to puke on
;;; itself, since it is the reference to the closure which normally causes the
;;; components to be combined.  This doesn't really cover all cases...
;;;
(defun delete-if-no-entries (component)
  (dolist (fun (component-lambdas component)
	       (delete-component component))
    (case (functional-kind fun)
      (:top-level (return))
      (:external
       (unless (every #'(lambda (ref)
			  (eq (block-component (node-block ref))
			      component))
		      (leaf-refs fun))
	 (return))))))

;;; COMPILE-COMPONENT -- internal.
;;;
(defun compile-component (component)
  (let* ((*compile-component* component)
ram's avatar
ram committed
	 (*byte-compiling*
	  (ecase *byte-compile*
	    ((t) t)
	    ((nil) nil)
	    (:maybe
	     (dolist (fun (component-lambdas component) t)
	       (unless (policy (lambda-bind fun)
			       (zerop speed) (<= debug 1))
		 (return nil)))))))

    (when *compile-print*
      (compiler-mumble "~&")
      (pprint-logical-block (*compiler-error-output* nil :per-line-prefix "; ")
	(compiler-mumble (intl:gettext "~:[~;Byte ~]Compiling ~A: ")
		       (component-name component))))
    (when *loop-analyze*
      (dfo-as-needed component)
      (maybe-mumble "Dom ")
      (find-dominators component)
      (maybe-mumble "Loop ")
      (loop-analyze component))


    (maybe-mumble "Env ")
    (environment-analyze component)
    (dfo-as-needed component)

    (when *record-xref-info*
      (maybe-mumble "[record-xref-info]~%")
      (record-component-xrefs component))

ram's avatar
ram committed
    (unless (eq (block-next (component-head component))
		(component-tail component))
      (if *byte-compiling*
	  (byte-compile-component component)
	  (native-compile-component component))))
  (when *compile-print*
    (compiler-mumble "~&"))

wlott's avatar
wlott committed
  (undefined-value))


;;;; Clearing global data structures:

;;; CLEAR-CONSTANT-INFO  --  Internal
wlott's avatar
wlott committed
;;;
;;;    Clear the INFO in constants in the *FREE-VARIABLES*, etc.  In addition
;;; to allowing stuff to be reclaimed, this is required for correct assignment
;;; of constant offsets, since we need to assign a new offset for each
;;; component.  We don't clear the FUNCTIONAL-INFO slots, since they are used
;;; to keep track of functions across component boundaries.
wlott's avatar
wlott committed
;;;
(defun clear-constant-info ()
wlott's avatar
wlott committed
  (maphash #'(lambda (k v)
	       (declare (ignore k))
	       (setf (leaf-info v) nil))
	   *constants*)

  (maphash #'(lambda (k v)
	       (declare (ignore k))
	       (when (constant-p v)
		 (setf (leaf-info v) nil)))
	   *free-variables*)

  (undefined-value))


;;; CLEAR-IR1-INFO  --  Internal
;;;
;;;    Blow away the REFS for all global variables, and recycle the IR1 for
;;; Component.
;;;
(defun clear-ir1-info (component)
  (declare (type component component))
  (labels ((blast (x)
	     (maphash #'(lambda (k v)
			  (declare (ignore k))
			  (when (leaf-p v)
			    (setf (leaf-refs v)
				  (delete-if #'here-p (leaf-refs v)))
			    (when (basic-var-p v)
			      (setf (basic-var-sets v)
				    (delete-if #'here-p (basic-var-sets v))))))
		      x))
	   (here-p (x)
	     (eq (block-component (node-block x)) component)))
    (blast *free-variables*)
    (blast *free-functions*)
    (blast *constants*))
  (macerate-ir1-component component)
  (undefined-value))


;;; CLEAR-STUFF  --  Interface
;;;
;;;    Clear all the global variables used by the compiler.
wlott's avatar
wlott committed
;;;
(defun clear-stuff (&optional (debug-too t))
wlott's avatar
wlott committed
  ;;
  ;; Clear global tables.
  (when (boundp '*free-functions*)
    (clrhash *free-functions*)
    (clrhash *free-variables*)
    (clrhash *constants*))

  (when debug-too
    (clrhash *continuation-numbers*)
    (clrhash *number-continuations*)
    (setq *continuation-number* 0)
    (clrhash *tn-ids*)
    (clrhash *id-tns*)
    (setq *tn-id* 0)
    (clrhash *label-ids*)
    (clrhash *id-labels*)
    (setq *label-id* 0)
    ;;
    ;; Clear some Pack data structures (for GC purposes only.)
    (assert (not *in-pack*))
    (dolist (sb (backend-sb-list *backend*))
      (when (finite-sb-p sb)
	(fill (finite-sb-live-tns sb) nil))))
wlott's avatar
wlott committed
  ;;
  ;; Reset Gensym.
  (setq lisp:*gensym-counter* 0)

  (values))
wlott's avatar
wlott committed

wlott's avatar
wlott committed
;;; PRINT-SUMMARY  --  Interface
;;;
;;;    This function is called by WITH-COMPILATION-UNIT at the end of a
;;; compilation unit.  It prints out any residual unknown function warnings and
;;; the total error counts.  ABORT-P should be true when the compilation unit
;;; was aborted by throwing out.  ABORT-COUNT is the number of dynamically
wlott's avatar
wlott committed
;;; enclosed nested compilation units that were aborted.
;;;
(defun print-summary (abort-p abort-count)
  (unless abort-p
    (handler-bind ((warning #'compiler-warning-handler))
      (let ((undefs (sort *undefined-warnings* #'string<
			  :key #'(lambda (x)
				   (let ((x (undefined-warning-name x)))
				     (if (symbolp x)
					 (symbol-name x)
					 (prin1-to-string x)))))))
	(unless *converting-for-interpreter*
	  (dolist (undef undefs)
	    (let ((name (undefined-warning-name undef))
		  (kind (undefined-warning-kind undef))
                  (context (undefined-warning-context undef))
		  (warnings (undefined-warning-warnings undef))
		  (count (undefined-warning-count undef)))
	      (dolist (*compiler-error-context* warnings)
		(compiler-warning _N"Undefined ~(~A~) ~S~@[ ~A~]" kind name context))
	      
	      (let ((warn-count (length warnings)))
		(when (and warnings (> count warn-count))
		  (let ((more (- count warn-count)))
		    (compiler-warning _N"~D more use~:P of undefined ~(~A~) ~S."
				      more kind name)))))))
	
	(dolist (kind '(:variable :function :type))
	  (let ((summary (mapcar #'undefined-warning-name
				 (remove kind undefs :test-not #'eq
					 :key #'undefined-warning-kind))))
	    (when summary
	      (compiler-warning
	       _N"~:[This ~(~A~) is~;These ~(~A~)s are~] undefined:~
		~%  ~{~<~%  ~1:;~S~>~^ ~}"
	       (cdr summary) kind summary)))))))
  (unless (or *converting-for-interpreter*
	      (and (not abort-p) (zerop abort-count)
		   (zerop *compiler-error-count*)
		   (zerop *compiler-warning-count*)
		   (zerop *compiler-note-count*)))
     (intl:gettext "~2&; Compilation unit ~:[finished~;aborted~].~
      ~[~:;~:*~&;   ~D fatal error~:P~]~
      ~[~:;~:*~&;   ~D error~:P~]~
      ~[~:;~:*~&;   ~D warning~:P~]~
      ~[~:;~:*~&;   ~D note~:P~]~2%")
     abort-p
     abort-count
     *compiler-error-count*
     *compiler-warning-count*
     *compiler-note-count*)))
wlott's avatar
wlott committed
;;; Describe-Component  --  Internal
;;;
;;;    Print out some useful info about Component to Stream.
;;;
(defun describe-component (component *standard-output*)
  (declare (type component component))
  (format t (intl:gettext "~|~%;;;; Component: ~S~2%") (component-name component))
  (print-blocks component)  
  (undefined-value))


(defun describe-ir2-component (component *standard-output*)
  (format t (intl:gettext "~%~|~%;;;; IR2 component: ~S~2%") (component-name component))
wlott's avatar
wlott committed
  
  (format t (intl:gettext "Entries:~%"))
  (dolist (entry (ir2-component-entries (component-info component)))
    (format t (intl:gettext "~4TL~D: ~S~:[~; [Closure]~]~%")
	    (label-id (entry-info-offset entry))
	    (entry-info-name entry)
	    (entry-info-closure-p entry)))
wlott's avatar
wlott committed
  
  (terpri)
  (pre-pack-tn-stats component *standard-output*)
  (terpri)
  (print-ir2-blocks component)
  (terpri)
wlott's avatar
wlott committed
  (undefined-value))


;;;; File reading:
;;;
;;;    When reading from a file, we have to keep track of some source
;;; information.  We also exploit our ability to back up for printing the error
;;; context and for recovering from errors.
;;;
;;; The interface we provide to this stuff is the stream-oid Source-Info
;;; structure.  The bookkeeping is done as a side-effect of getting the next
;;; source form.


;;; The File-Info structure holds all the source information for a given file.
;;;
(defstruct file-info
  ;;
  ;; If a file, the truename of the corresponding source file.  If from a Lisp
  ;; form, :LISP, if from a stream, :STREAM.
  (name (required-argument) :type (or pathname (member :lisp :stream)))
wlott's avatar
wlott committed
  ;;
  ;; The defaulted, but not necessarily absolute file name (i.e. prior to
  ;; TRUENAME call.)  Null if not a file.  This is used to set
  ;; *COMPILE-FILE-PATHNAME*, and if absolute, is dumped in the debug-info.
  (untruename nil :type (or pathname null))
wlott's avatar
wlott committed
  ;; The file's write date (if relevant.)
  (write-date nil :type (or unsigned-byte null))
  ;;
ram's avatar
ram committed
  ;; This file's FILE-COMMENT, or NIL if none.
  (comment nil :type (or simple-string null))
  ;;
wlott's avatar
wlott committed
  ;; The source path root number of the first form in this file (i.e. the
  ;; total number of forms converted previously in this compilation.)
  (source-root 0 :type unsigned-byte)
  ;;
  ;; Parallel vectors containing the forms read out of the file and the file
  ;; positions that reading of each form started at (i.e. the end of the
  ;; previous form.)
  (forms (make-array 10 :fill-pointer 0 :adjustable t) :type (vector t))
  (positions (make-array 10 :fill-pointer 0 :adjustable t) :type (vector t))
  ;;
  ;; Language to use.  Normally Lisp, but sometimes Dylan.
  (language :lisp :type (member :lisp #+nil :dylan)))
wlott's avatar
wlott committed

;;; The Source-Info structure provides a handle on all the source information
;;; for an entire compilation.
;;;
(defstruct (source-info
	    (:print-function
	     (lambda (s stream d)
	       (declare (ignore s d))
	       (format stream "#<Source-Info>"))))
  ;;
  ;; The UT that compilation started at.
  (start-time (get-universal-time) :type unsigned-byte)
  ;;
  ;; A list of the file-info structures for this compilation.
  (files nil :type list)
  ;;
  ;; The tail of the Files for the file we are currently reading.
  (current-file nil :type list)
  ;;
  ;; The stream that we are using to read the Current-File.  Null if no stream
  ;; has been opened yet.
  (stream nil :type (or stream null))
  ;;
  ;; External format to use for the stream if the stream hasn't been opened
  #+unicode
  (external-format :default)
  ;;
  ;; How to handle decoding errors when reading the source file.
  ;; Default is T to signal an error.
  #+unicode
  (decoding-error t))
wlott's avatar
wlott committed


;;; Make-File-Source-Info  --  Internal
;;;
;;;    Given a list of pathnames, return a Source-Info structure.
;;;
(defun make-file-source-info (files external-format decoding-error)
wlott's avatar
wlott committed
  (declare (list files))
  (let ((file-info
	 (mapcar #'(lambda (x)
ram's avatar
ram committed
		     (make-file-info :name (truename x)
				     :write-date (file-write-date x)
wlott's avatar
wlott committed
		 files)))
    (make-source-info :files file-info
		      :current-file file-info
		      #+unicode :external-format
		      #+unicode :decoding-error
		      #+unicode decoding-error)))
wlott's avatar
wlott committed


;;; MAKE-LISP-SOURCE-INFO  --  Interface
;;;
;;;    Return a SOURCE-INFO to describe the incremental compilation of Form.
;;; Also used by EVAL:INTERNAL-EVAL.
;;;
(defun make-lisp-source-info (form)
  (make-source-info
   :start-time (get-universal-time)
   :files (list (make-file-info :name :lisp
				:forms (vector form)
				:positions '#(0)))))


;;; MAKE-STREAM-SOURCE-INFO  --  Internal
;;;
;;;    Return a SOURCE-INFO which will read from Stream.
;;;
(defun make-stream-source-info (stream language)
  (declare (type (member :lisp) language))
  (let ((files (list (make-file-info :name :stream :language language))))
wlott's avatar
wlott committed
    (make-source-info
     :files files
     :current-file files
     :stream stream)))


;;; Normal-Read-Error  --  Internal
;;;
;;;    Print an error message for a non-EOF error on Stream.  Old-Pos is a
;;; preceding file position that hopefully comes before the beginning of the
;;; line.  Of course, this only works on streams that support the file-position
;;; operation.
;;;
(defun normal-read-error (stream old-pos condition)
  (declare (type stream stream) (type unsigned-byte old-pos))
wlott's avatar
wlott committed
  (let ((pos (file-position stream)))
    (file-position stream old-pos)
    (let ((start old-pos))
      (loop
	(let ((line (read-line stream nil))
	      (end (file-position stream)))
	  (when (>= end pos)
	     pos (intl:gettext "Read error at ~D:~% \"~A/\\~A\"~%~A")
	     pos (string-left-trim '(#\space #\tab)
				   (subseq line 0 (- pos start)))
wlott's avatar
wlott committed
	     (subseq line (- pos start))
	     condition)
	    (return))
	  (setq start end)))))
  (undefined-value))


;;; Ignore-Error-Form  --  Internal
;;;
;;;    Back Stream up to the position Pos, then read a form with
;;; *Read-Suppress* on, discarding the result.  If an error happens during this
;;; read, then bail out using Compiler-Error (fatal in this context).
;;;
(defun ignore-error-form (stream pos)
wlott's avatar
wlott committed
  (declare (type stream stream) (type unsigned-byte pos))
  (file-position stream pos)
  (handler-case (let ((*read-suppress* t)
		      (*features* (backend-features *target-backend*)))
wlott's avatar
wlott committed
    (error (condition)
      (declare (ignore condition))
      (compiler-error _N"Unable to recover from read error."))))
wlott's avatar
wlott committed


;;; Unexpected-EOF-Error  --  Internal
;;;
;;;    Print an error message giving some context for an EOF error.  We print
;;; the first line after POS that contains #\" or #\(, or lacking that, the
wlott's avatar
wlott committed
;;; first non-empty line.
;;;
(defun unexpected-eof-error (stream pos condition)
  (declare (type stream stream) (type unsigned-byte pos))
  (let ((eof-pos (file-position stream))
        (res nil))
wlott's avatar
wlott committed
    (file-position stream pos)
    (loop
       (let ((line (read-line stream nil nil))) 
	 (unless line (return))
	 (when (or (find #\" line) (find #\( line))
	   (setq res line)
	   (return))
	 (unless (or res (zerop (length line)))
	   (setq res line))))
wlott's avatar
wlott committed

     pos _"Read error in form starting at ~D:~%~@[ \"~A\"~%~]~A"
     pos res condition)
wlott's avatar
wlott committed

    (file-position stream eof-pos)
    (undefined-value)))
wlott's avatar
wlott committed


;;; Careful-Read  --  Internal
;;;
;;;    Read a form from STREAM, returning EOF at EOF.  If a read error happens,
wlott's avatar
wlott committed
;;; then attempt to recover if possible, returing a proxy error form.
;;;
(defun careful-read (stream eof pos)
  (handler-case (let ((*features* (backend-features *target-backend*)))
wlott's avatar
wlott committed
    (error (condition)
      (if (null (peek-char nil stream nil))
          (unexpected-eof-error stream pos condition)
          (progn
            (normal-read-error stream pos condition)
            (ignore-error-form stream pos)))
      '(cerror (intl:gettext "Skip this form.")
	       (intl:gettext "Attempt to load a file having a compile-time read error.")))))
wlott's avatar
wlott committed


;;; Get-Source-Stream  --  Internal
;;;
;;;    If Stream is present, return it, otherwise open a stream to the current
;;; file.  There must be a current file.  When we open a new file, we also
;;; reset *Package* and policy.  This gives the effect of rebinding
wlott's avatar
wlott committed
;;; around each file.
;;;
(defun get-source-stream (info)
  (declare (type source-info info))
  (cond ((source-info-stream info))
	(t
	 (setq *package* *initial-package*)
	 (setq *default-cookie* (copy-cookie *initial-cookie*))
	 (setq *default-interface-cookie*
	       (copy-cookie *initial-interface-cookie*))
	 (let* ((finfo (first (source-info-current-file info)))
		(name (file-info-name finfo)))
	   (setq *compile-file-truename* name)
	   (setq *compile-file-pathname* (file-info-untruename finfo))
	   (setf (source-info-stream info)
		 (open name :direction :input
		       #+unicode :external-format
		       #+unicode (source-info-external-format info)
		       #+unicode :decoding-error
		       #+unicode (source-info-decoding-error info)))))))
wlott's avatar
wlott committed

;;; CLOSE-SOURCE-INFO  --  Internal
;;;
;;;    Close the stream in Info if it is open.
;;;
(defun close-source-info (info)
  (declare (type source-info info))
  (let ((stream (source-info-stream info)))
    (when stream (close stream)))
  (setf (source-info-stream info) nil)
  (undefined-value))


;;; Advance-Source-File  --  Internal
;;;
;;;    Advance Info to the next source file.  If none, return NIL, otherwise T.
;;;
(defun advance-source-file (info)
  (declare (type source-info info))
  (close-source-info info)
  (let ((prev (pop (source-info-current-file info))))
    (if (source-info-current-file info)
	(let ((current (first (source-info-current-file info))))
	  (setf (file-info-source-root current)
		(+ (file-info-source-root prev)
		   (length (file-info-forms prev))))
	  t)
	nil)))

;;; PROCESS-SOURCES -- internal.
wlott's avatar
wlott committed
;;;
;;; Read the sources from the source files and process them.
;;; 
(defun process-sources (info)
  (let* ((file (first (source-info-current-file info)))
	 (language (file-info-language file))
	 (stream (get-source-stream info)))
    (ecase language
      (:lisp
rtoy's avatar
rtoy committed
       (flet
	   ((process-one (stream)
	      (loop
		 (let* ((pos (file-position stream))
			(eof '(*eof*))
			(form (careful-read stream eof pos)))
		   (if (eq form eof)
		       (return)
		       (let* ((forms (file-info-forms file))
			      (current-idx (+ (fill-pointer forms)
					      (file-info-source-root file))))
			 (vector-push-extend form forms)
			 (vector-push-extend pos (file-info-positions file))
			 (clrhash *source-paths*)
			 (find-source-paths form current-idx)
			 (process-form form
				       `(original-source-start 0 ,current-idx)))))))
	    (process-xref-info (pathname)
	      ;; When we have xref enabled, we save the xref data to the
	      ;; file by faking it.  What we do is append a bunch of forms
	      ;; to the file as if the file actually contained them.  These
	      ;; forms clear out the entries from the xref databases
	      ;; pertaining to this file, and then registers new entries
	      ;; based on what we've found out so far from compiling this
	      ;; file.
	      ;;
	      ;; Is this what we really want to do?  A new FOP might be good.
	      ;;
	      ;; Also, when compiling a file should we have a file-local
	      ;; version of the databases?  This makes it easy to figure
	      ;; out what we need to save to the fasl.  Then we can update
	      ;; the global tables with the new info when we're done.  Or
	      ;; we can just wait until the user loads the fasl.  This
	      ;; latter option, however, changes how xref currently behaves.

	      ;; Set *compile-print* to nil so we don't see any
	      ;; spurious output from our fake source forms.  (Do we
	      ;; need more?)
	      (let ((*compile-print* nil))
		;; Clear the xref database of all references to this file
		(when (or (pathnamep pathname)
			  (stringp pathname))
		  (process-form `(xref::invalidate-xrefs-for-namestring
				  ,(namestring pathname))
				;; What should we use here?
				`(original-source-start 0 0))
		  ;; Now dump all the xref info pertaining to this file
		  (dolist (db-type '(:calls :called :references :binds :sets
				     :macroexpands))
		    (dolist (xrefs (xref::find-xrefs-for-pathname db-type pathname))
		      (destructuring-bind (target contexts)
			  xrefs
			(dolist (c contexts)
			  (process-form
			   `(xref:register-xref
			     ,db-type ',target
			     (xref:make-xref-context :name ',(xref:xref-context-name c)
						     :file ,(xref:xref-context-file c)
						     :source-path ',(xref:xref-context-source-path c)))

			   `(original-source-start 0 0))))))))))
	 ;; Compile the real file.
	 (process-one stream)

	 (when *record-xref-info*
	   (process-xref-info (file-info-name file))))
      
       (when (advance-source-file info)
	 (process-sources info))))))
wlott's avatar
wlott committed

;;; FIND-FILE-INFO  --  Interface
;;;