From 1a5310b881b9a4532eb3bfea6a24f8019b0561f0 Mon Sep 17 00:00:00 2001 From: Justin Ethier Date: Tue, 10 Aug 2021 22:43:52 -0400 Subject: [PATCH 1/7] Stage breaking up emitting C file / compiling C --- cyclone.scm | 77 ++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 56 insertions(+), 21 deletions(-) diff --git a/cyclone.scm b/cyclone.scm index 6ab58f1e..53079e80 100644 --- a/cyclone.scm +++ b/cyclone.scm @@ -754,17 +754,6 @@ " " lib-options) lib-options))) - ;; Only read C compiler options from module being compiled - (cc-opts* - (cond - (program? - (string-join ;; Check current program for options - (program-c-compiler-opts! in-prog) - " ")) - (else - (string-join - (lib:c-compiler-options (car in-prog)) - " ")))) (exec-file (basename in-file)) (src-file (string-append exec-file ".c")) (meta-file (string-append exec-file ".meta")) @@ -803,9 +792,57 @@ lib-deps) in-file append-dirs - prepend-dirs))))) - (result (create-c-file in-prog))) + prepend-dirs)))))) + (create-c-file in-prog) + (when (not program?) + ;; Emit .meta file + (with-output-to-file + meta-file + (lambda () + (display ";; This file was automatically generated by the Cyclone Scheme compiler") + (newline) + (write (macro:get-defined-macros))))))) +(define (run-external-compiler args cc? cc-prog cc-exec cc-lib cc-so + cc-opts cc-prog-linker-opts cc-prog-linker-objs + append-dirs prepend-dirs) + (let* ((in-file (car args)) + (expander (base-expander)) + (in-prog-raw (read-file in-file)) + (program? (not (library? (car in-prog-raw)))) + (in-prog + (cond + (program? + (Cyc-add-feature! 'program) ;; Load special feature + ;; TODO: what about top-level cond-expands in the program? + in-prog-raw) + (else + ;; Account for any cond-expand declarations in the library + (list (lib:cond-expand (car in-prog-raw) expander))))) + ;; how to collect these? would it be best if we write them to file + ;; and then we pick that file back up now? + (c-linker-options 'todo) + ;; similar to above, may need to emit these and pick them back up now from file + (lib-deps 'todo) + ;; Only read C compiler options from module being compiled + (cc-opts* + (cond + (program? + (string-join ;; Check current program for options + (program-c-compiler-opts! in-prog) + " ")) + (else + (string-join + (lib:c-compiler-options (car in-prog)) + " ")))) + (exec-file (basename in-file)) + (src-file (string-append exec-file ".c")) + (get-comp-env + (lambda (sym str) + (if (> (string-length str) 0) + str + (Cyc-compilation-environment sym)))) + ) ;; Compile the generated C file (cond (program? @@ -857,13 +894,6 @@ (display comp-objs-cmd) (newline))))) (else - ;; Emit .meta file - (with-output-to-file - meta-file - (lambda () - (display ";; This file was automatically generated by the Cyclone Scheme compiler") - (newline) - (write (macro:get-defined-macros)))) ;; Compile library (let ((comp-lib-cmd (string-append @@ -1075,5 +1105,10 @@ Debug options: (exit 1))) (run-compiler non-opts compile? cc-prog cc-exec cc-lib cc-so cc-opts cc-linker-opts cc-linker-extra-objects - append-dirs prepend-dirs))))) + append-dirs prepend-dirs) + (run-external-compiler non-opts compile? cc-prog cc-exec cc-lib cc-so + cc-opts cc-linker-opts cc-linker-extra-objects + append-dirs prepend-dirs) + + )))) From 1d0cbf96ed02494d63ae3867b9bd0532c23d5c8b Mon Sep 17 00:00:00 2001 From: Justin Ethier Date: Tue, 17 Aug 2021 13:41:48 -0400 Subject: [PATCH 2/7] Use meta file to pass data when compiling programs --- cyclone.scm | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/cyclone.scm b/cyclone.scm index 53079e80..dff72294 100644 --- a/cyclone.scm +++ b/cyclone.scm @@ -794,14 +794,38 @@ append-dirs prepend-dirs)))))) (create-c-file in-prog) - (when (not program?) + (cond + (program? + ;; Use .meta file to store information for C compiler phase + (save-program-metadata meta-file lib-deps c-linker-options)) + (else ;; Emit .meta file (with-output-to-file meta-file (lambda () (display ";; This file was automatically generated by the Cyclone Scheme compiler") (newline) - (write (macro:get-defined-macros))))))) + (write (macro:get-defined-macros)))))))) + +(define (save-program-metadata filename lib-deps c-linker-options) + (with-output-to-file + filename + (lambda () + (display ";; This file was automatically generated by the Cyclone Scheme compiler") + (newline) + (write `(lib-deps . ,lib-deps)) + (newline) + (write `(c-linker-options . ,c-linker-options))))) + +(define (load-program-metadata filename) + (let ((data (call-with-input-file filename read-all))) + (delete-file filename) + data)) + +(define (get-meta meta symbol default) + (if (assoc symbol meta) + (cdr (assoc symbol meta)) + default)) (define (run-external-compiler args cc? cc-prog cc-exec cc-lib cc-so cc-opts cc-prog-linker-opts cc-prog-linker-objs @@ -819,11 +843,6 @@ (else ;; Account for any cond-expand declarations in the library (list (lib:cond-expand (car in-prog-raw) expander))))) - ;; how to collect these? would it be best if we write them to file - ;; and then we pick that file back up now? - (c-linker-options 'todo) - ;; similar to above, may need to emit these and pick them back up now from file - (lib-deps 'todo) ;; Only read C compiler options from module being compiled (cc-opts* (cond @@ -837,6 +856,7 @@ " ")))) (exec-file (basename in-file)) (src-file (string-append exec-file ".c")) + (meta-file (string-append exec-file ".meta")) (get-comp-env (lambda (sym str) (if (> (string-length str) 0) @@ -846,7 +866,10 @@ ;; Compile the generated C file (cond (program? - (letrec ((objs-str + (letrec ((metadata (load-program-metadata meta-file)) + (c-linker-options (get-meta metadata 'c-linker-options '())) + (lib-deps (get-meta metadata 'lib-deps '())) + (objs-str (string-append cc-prog-linker-objs (apply From 08c4e8f2e62093f873fabc8faa71d21e22fdbb15 Mon Sep 17 00:00:00 2001 From: Justin Ethier Date: Mon, 16 Aug 2021 20:24:36 -0400 Subject: [PATCH 3/7] Remove unused args --- cyclone.scm | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/cyclone.scm b/cyclone.scm index dff72294..39898ad9 100644 --- a/cyclone.scm +++ b/cyclone.scm @@ -721,9 +721,7 @@ in-prog)) ;; Compile and emit: -(define (run-compiler args cc? cc-prog cc-exec cc-lib cc-so - cc-opts cc-prog-linker-opts cc-prog-linker-objs - append-dirs prepend-dirs) +(define (run-compiler args append-dirs prepend-dirs) (let* ((in-file (car args)) (expander (base-expander)) (in-prog-raw (read-file in-file)) @@ -827,9 +825,10 @@ (cdr (assoc symbol meta)) default)) -(define (run-external-compiler args cc? cc-prog cc-exec cc-lib cc-so - cc-opts cc-prog-linker-opts cc-prog-linker-objs - append-dirs prepend-dirs) +(define (run-external-compiler + args append-dirs prepend-dirs + cc? cc-prog cc-exec cc-lib cc-so + cc-opts cc-prog-linker-opts cc-prog-linker-objs) (let* ((in-file (car args)) (expander (base-expander)) (in-prog-raw (read-file in-file)) @@ -1126,12 +1125,10 @@ Debug options: (cdr err)) (newline) (exit 1))) - (run-compiler non-opts compile? cc-prog cc-exec cc-lib cc-so - cc-opts cc-linker-opts cc-linker-extra-objects - append-dirs prepend-dirs) - (run-external-compiler non-opts compile? cc-prog cc-exec cc-lib cc-so - cc-opts cc-linker-opts cc-linker-extra-objects - append-dirs prepend-dirs) + (run-compiler non-opts append-dirs prepend-dirs) + (run-external-compiler non-opts compile? append-dirs prepend-dirs + cc-prog cc-exec cc-lib cc-so + cc-opts cc-linker-opts cc-linker-extra-objects) )))) From 50631b8bb5814f85604af5b2f1963495ca3a302d Mon Sep 17 00:00:00 2001 From: Justin Ethier Date: Mon, 16 Aug 2021 22:22:04 -0400 Subject: [PATCH 4/7] Use a separate thread to emit the C file --- cyclone.scm | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/cyclone.scm b/cyclone.scm index 39898ad9..66fc086b 100644 --- a/cyclone.scm +++ b/cyclone.scm @@ -21,7 +21,8 @@ (scheme cyclone primitives) (scheme cyclone transforms) (scheme cyclone cps-optimizations) - (scheme cyclone libraries)) + (scheme cyclone libraries) + (srfi 18)) (define *fe:batch-compile* #t) ;; Batch compilation. TODO: default to false or true?? (define *optimization-level* 2) ;; Default level @@ -1125,10 +1126,13 @@ Debug options: (cdr err)) (newline) (exit 1))) - (run-compiler non-opts append-dirs prepend-dirs) - (run-external-compiler non-opts compile? append-dirs prepend-dirs - cc-prog cc-exec cc-lib cc-so - cc-opts cc-linker-opts cc-linker-extra-objects) + (let ((t (thread-start! + (make-thread (lambda () (run-compiler non-opts append-dirs prepend-dirs)))))) + (thread-join! t) + (run-external-compiler + non-opts compile? append-dirs prepend-dirs + cc-prog cc-exec cc-lib cc-so + cc-opts cc-linker-opts cc-linker-extra-objects)) )))) From 7d92a39fdf26c919102eab879b02e55a2cb983f8 Mon Sep 17 00:00:00 2001 From: Justin Ethier Date: Mon, 16 Aug 2021 23:34:00 -0400 Subject: [PATCH 5/7] Run scm compiler as a sub-process --- cyclone.scm | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/cyclone.scm b/cyclone.scm index 66fc086b..052819ab 100644 --- a/cyclone.scm +++ b/cyclone.scm @@ -983,6 +983,7 @@ ; (equal? #\- (string-ref arg 0))))) ; args)) (compile? #t) + (run-scm-compiler? (member "-run-scm-compiler" args)) (cc-prog (apply string-append (collect-opt-values args "-CP"))) (cc-exec (apply string-append (collect-opt-values args "-CE"))) (cc-lib (apply string-append (collect-opt-values args "-CL"))) @@ -1126,13 +1127,20 @@ Debug options: (cdr err)) (newline) (exit 1))) - (let ((t (thread-start! - (make-thread (lambda () (run-compiler non-opts append-dirs prepend-dirs)))))) - (thread-join! t) - (run-external-compiler - non-opts compile? append-dirs prepend-dirs - cc-prog cc-exec cc-lib cc-so - cc-opts cc-linker-opts cc-linker-extra-objects)) - + (cond + (run-scm-compiler? + ;; Compile Scheme code into a C file + (run-compiler non-opts append-dirs prepend-dirs)) + (else + ;; Generate the C file + (system + (string-append + (calling-program) " -run-scm-compiler " + (string-join args " "))) + ;; Call the C compiler + (run-external-compiler + non-opts compile? append-dirs prepend-dirs + cc-prog cc-exec cc-lib cc-so + cc-opts cc-linker-opts cc-linker-extra-objects))) )))) From 99ce726ca319da8f4aaf0573d8507662d5a5f7ff Mon Sep 17 00:00:00 2001 From: Justin Ethier Date: Mon, 16 Aug 2021 23:39:09 -0400 Subject: [PATCH 6/7] Fix order of args passed to run-external-compiler --- cyclone.scm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cyclone.scm b/cyclone.scm index 052819ab..329458b6 100644 --- a/cyclone.scm +++ b/cyclone.scm @@ -1139,8 +1139,8 @@ Debug options: (string-join args " "))) ;; Call the C compiler (run-external-compiler - non-opts compile? append-dirs prepend-dirs - cc-prog cc-exec cc-lib cc-so + non-opts append-dirs prepend-dirs + compile? cc-prog cc-exec cc-lib cc-so cc-opts cc-linker-opts cc-linker-extra-objects))) )))) From 876a93bf392253fffc6e737f411717f430ea841f Mon Sep 17 00:00:00 2001 From: Justin Ethier Date: Tue, 17 Aug 2021 05:04:36 -0400 Subject: [PATCH 7/7] Added -no-compiler-subprocess option --- cyclone.scm | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/cyclone.scm b/cyclone.scm index 329458b6..30f26569 100644 --- a/cyclone.scm +++ b/cyclone.scm @@ -984,6 +984,7 @@ ; args)) (compile? #t) (run-scm-compiler? (member "-run-scm-compiler" args)) + (no-compiler-subprocess (member "-no-compiler-subprocess" args)) (cc-prog (apply string-append (collect-opt-values args "-CP"))) (cc-exec (apply string-append (collect-opt-values args "-CE"))) (cc-lib (apply string-append (collect-opt-values args "-CL"))) @@ -1133,10 +1134,18 @@ Debug options: (run-compiler non-opts append-dirs prepend-dirs)) (else ;; Generate the C file - (system - (string-append - (calling-program) " -run-scm-compiler " - (string-join args " "))) + (cond + (no-compiler-subprocess + ;; Special case, we can generate .C file within this process + (run-compiler non-opts append-dirs prepend-dirs)) + (else + ;; Normal path is to run another instance of cyclone to generate + ;; the .C file. This lets us immediately free those resources once + ;; the Scheme compilation is done. + (system + (string-append + (calling-program) " -run-scm-compiler " + (string-join args " "))))) ;; Call the C compiler (run-external-compiler non-opts append-dirs prepend-dirs