add includes

README.md

@@ -136,6 +136,29 @@ The same program can be written in a compact form:
 ((ld r0 100)(:again)(sub r0 1 (nz? (j :again))))
 ```
 
+### Includes
+
+Croissant allows program composition using includes.
+
+```
+(include path)
+```
+
+Path can be a filesystem absolute or relative path (with slashes). Relative path is always resolved from the current source file.
+Use double quotes if needed.
+
+If the included file is missing an extension, `.csn` is appended automatically.
+
+Each included file must contain a list of instructions or procedures, just like the main file.
+
+Includes work at the S-expression level. The included file is parsed to instructions and inserted
+in place of the include directive. Error reports will always show which file the line and column refer to.
+
+There are no special scoping rules. It is possible to define a label in a file and jump to it from another.
+Defines and register aliases (sym) also work cross-file. 
+
+Don't go too crazy with includes, it can get messy. Or do, I'm not your mom.
+
 ## Instruction
 
 Instructions are written like this:

crsn/crsn-sexp/src/error.rs

@@ -23,7 +23,7 @@ pub(crate) type ERes<T> = Result<T, Err>;
 
 impl fmt::Display for Error {
     fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
-        write!(f, "{}:{}: {}", self.pos.line, self.pos.column, self.message)
+        write!(f, "{}: {}", self.pos, self.message)
     }
 }
 

crsn/crsn-sexp/src/position.rs

@@ -20,6 +20,13 @@ impl fmt::Display for SourcePosition {
     }
 }
 
+impl SourcePosition {
+    pub fn with_file(mut self, file: u32) -> Self {
+        self.file = file;
+        self
+    }
+}
+
 impl Debug for SourcePosition {
     fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
         if f.alternate() {
@@ -31,7 +38,7 @@ impl Debug for SourcePosition {
                 .finish()
         } else {
             // shorter version
-            write!(f, "Pos({}:{})", self.line, self.column)
+            write!(f, "Pos({}|{}:{})", self.file, self.line, self.column)
         }
     }
 }

crsn/src/asm/error.rs

@@ -8,6 +8,7 @@ use sexp::SourcePosition;
 use crate::asm::data::{Register};
 use crate::asm::data::literal::Label;
 use crate::asm::instr::Cond;
+use std::io;
 
 /// csn_asm unified error type
 #[derive(Error, Debug)]
@@ -20,6 +21,21 @@ pub enum CrsnError {
     ParseOther(Box<dyn Error + Send + Sync>, SourcePosition),
     #[error("Assembler error: {0} at {1}")]
     Asm(AsmError, SourcePosition),
+    #[error("IO error: {0}")]
+    IOError(#[from] io::Error),
+}
+
+impl CrsnError {
+    /// Get error pos
+    pub fn pos(&self) -> Option<&SourcePosition> {
+        match self {
+            CrsnError::Parse(_, p) => Some(p),
+            CrsnError::ParseOther(_, p) => Some(p),
+            CrsnError::Asm(_, p) => Some(p),
+            CrsnError::Sexp(se) => Some(&se.pos),
+            CrsnError::IOError(_) =>None,
+        }
+    }
 }
 
 /// Error from the assembler stage (after parsing S-expressions and basic validation)

crsn/src/asm/mod.rs

@@ -11,26 +11,25 @@ use crate::builtin::BuiltinOps;
 use crate::runtime::run_thread::{RunState, ThreadInfo, ThreadToken};
 use crate::runtime::frame::REG_COUNT;
 use std::sync::atomic::AtomicU32;
+use std::path::{Path};
 
 pub mod data;
 pub mod error;
 pub mod instr;
 pub mod parse;
 pub mod patches;
+mod read_file;
+use read_file::read_file;
 
-/// Parse a program from string and assemble a low level instruction sequence from it.
-pub fn assemble(source: &str, uniq : &CrsnUniq, mut parsers: Vec<Box<dyn CrsnExtension>>) -> Result<Arc<Program>, error::CrsnError> {
-    parsers.insert(0, BuiltinOps::new());
-    for p in &mut parsers {
-        p.init(uniq);
-    }
+pub(crate) fn read_source_file(path: impl AsRef<Path>) -> Result<String, error::CrsnError> {
+    trace!("Read source file: {}", path.as_ref().display());
 
-    let parsers_arc = Arc::new(parsers);
+    let source = read_file(path)?;
 
     // remove first line if it looks like a shebang
-    let source = if source.starts_with("#!") {
+    let s = if source.starts_with("#!") {
         if let Some(nl) = source.find('\n') {
-            &source[nl + 1..]
+            (&source[nl + 1..]).to_string()
         } else {
             source
         }
@@ -38,10 +37,25 @@ pub fn assemble(source: &str, uniq : &CrsnUniq, mut parsers: Vec<Box<dyn CrsnExt
         source
     };
 
+    Ok(s)
+}
+
+/// Parse a program from string and assemble a low level instruction sequence from it.
+pub fn assemble(path: impl AsRef<Path>, uniq : &CrsnUniq, mut parsers: Vec<Box<dyn CrsnExtension>>) -> Result<Arc<Program>, error::CrsnError> {
+    parsers.insert(0, BuiltinOps::new());
+    for p in &mut parsers {
+        p.init(uniq);
+    }
+
+    let path = path.as_ref().canonicalize()?;
+    let source = read_source_file(&path)?;
+
+    let parsers_arc = Arc::new(parsers);
+
     let ti = Arc::new(ThreadInfo {
         id: ThreadToken(0),
         uniq: Default::default(),
-        program: Program::new(vec![], parsers_arc.clone()).unwrap(),
+        program: Program::new(vec![], parsers_arc.clone(), vec![path.clone()]).unwrap(),
         cycle_time: Default::default(),
         scheduler_interval: Default::default(),
         extensions: parsers_arc.clone(),
@@ -72,12 +86,27 @@ pub fn assemble(source: &str, uniq : &CrsnUniq, mut parsers: Vec<Box<dyn CrsnExt
             },
             const_eval_ti: ti,
             parsing_expr: false,
-            label_num: label_num.clone()
+            label_num: label_num.clone(),
+            files: vec![
+                path,
+            ],
+            active_file: 0
         }),
     };
 
-    let ops = parse::parse(source, &SourcePosition::default(), &pcx)?;
+    let res = do_parse(&source, &pcx, parsers_arc.clone());
+    if let Err(e) = &res {
+        if let Some(pos) = e.pos() {
+            let f = pcx.state.borrow().files[pos.file as usize].clone();
+            eprintln!("Error in source file: {}", f.display());
+        }
+    }
+    res
+}
+
+fn do_parse(source: &str, pcx : &ParserContext, parsers_arc : Arc<Vec<Box<dyn CrsnExtension>>>) -> Result<Arc<Program>, error::CrsnError> {
+    let ops = parse::parse(source, &SourcePosition::default(), pcx)?;
     let ops = jumps_to_skips(ops)?;
 
-    Ok(Program::new(ops, parsers_arc)?)
+    Ok(Program::new(ops, parsers_arc, pcx.state.borrow_mut().files.split_off(0))?)
 }

crsn/src/asm/parse/mod.rs

@@ -13,6 +13,7 @@ use crate::asm::parse::sexp_expect::expect_list;
 use crate::module::CrsnExtension;
 use crate::runtime::run_thread::{ThreadInfo, RunState};
 use std::sync::Arc;
+use std::path::PathBuf;
 
 pub mod parse_cond;
 pub mod parse_instr;
@@ -54,6 +55,12 @@ pub struct ParserState {
 
     /// Label numberer
     pub label_num : Arc<AtomicU32>,
+
+    /// Numbered files
+    pub files : Vec<PathBuf>,
+
+    /// Active file number. Is backed up before descending into a sub-file, and restored afterwards.
+    pub active_file: usize,
 }
 
 impl ParserState {

crsn/src/asm/parse/parse_instr.rs

@@ -6,14 +6,19 @@ use crate::asm::parse::arg_parser::TokenParser;
 use crate::asm::parse::parse_cond::parse_cond_branch;
 use crate::asm::parse::parse_routine::parse_routine;
 use crate::asm::parse::ParserContext;
-use crate::asm::parse::sexp_expect::{expect_list, expect_string_atom};
+use crate::asm::parse::sexp_expect::{expect_list, expect_string_atom, expect_any_string_atom};
 use crate::asm::patches::NextOrErr;
 use crate::module::ParseRes;
+use crate::asm::patches::ErrSetFile;
 
 use super::parse_op::parse_op;
+use std::path::{PathBuf};
+use std::convert::TryFrom;
+use crate::asm::read_source_file;
+use crate::asm::instr::flatten::jumps_to_skips;
 
 pub fn parse_instructions(items: impl Iterator<Item=Sexp>, pos: &SourcePosition, pcx: &ParserContext) -> Result<Box<dyn Flatten>, CrsnError> {
-    let mut parsed = vec![];
+    let mut parsed: Vec<Box<dyn Flatten>> = vec![];
     'exprs: for expr in items {
         let (tokens, listpos) = expect_list(expr, false)?;
 
@@ -28,6 +33,84 @@ pub fn parse_instructions(items: impl Iterator<Item=Sexp>, pos: &SourcePosition,
             }
         }
 
+        if name == "include" {
+            // TODO move this to a separate function and get rid of err_file()
+
+            if parsing_expr {
+                return Err(CrsnError::Parse("Illegal syntax in const expression".into(), pos.clone()));
+            }
+
+            let (mut path, _namepos) = expect_any_string_atom(toki.next_or_err(listpos.clone(), "Expected file path or name to include")?)?;
+
+            // add extension if missing
+            let last_piece = path.split('/').rev().next().unwrap();
+            if !last_piece.contains('.') {
+                path.push_str(".csn");
+            }
+
+            trace!("*** include, raw path: {}", path);
+
+            let state = pcx.state.borrow();
+            let this_file = &state.files[state.active_file];
+
+            let new_file = PathBuf::try_from(path).unwrap();
+
+            let new_pb = if !new_file.is_absolute() {
+                let parent = this_file.parent().expect("file has parent");
+
+                let fixed = parent.join(&new_file);
+                trace!("Try to resolve: {}", fixed.display());
+
+                fixed.canonicalize()?
+            } else {
+                new_file.to_owned()
+            };
+            drop(state);
+            drop(new_file);
+
+            let (old_af, af) = {
+                let mut state = pcx.state.borrow_mut();
+                let old_af = state.active_file;
+                let af = state.files.len();
+                state.active_file = af;
+                state.files.push(new_pb.clone());
+                (old_af, af as u32)
+            };
+
+            let loaded = read_source_file(&new_pb)
+                .err_file(af)?;
+
+            let (items, _pos) =
+                expect_list(
+                    sexp::parse(&loaded)
+                        .map_err(|mut e| {
+                            e.pos.file = af;
+                            e
+                        })?,
+                    true)
+                .err_file(af)?;
+
+            let sub_parsed = parse_instructions(items.into_iter(), pos, pcx)
+                .err_file(af)?;
+
+            let mut flat = sub_parsed.flatten(&pcx.state.borrow().label_num)
+                .err_file(af)?;
+
+            flat.iter_mut().for_each(|op| {
+                op.pos.file = af as u32;
+            });
+
+            trace!("inner falt {:#?}", flat);
+
+            parsed.push(Box::new(flat));
+
+            {
+                let mut state = pcx.state.borrow_mut();
+                state.active_file = old_af;
+            }
+            continue;
+        }
+
         if name == "proc" {
             if parsing_expr {
                 return Err(CrsnError::Parse("Illegal syntax in const expression".into(), pos.clone()));

crsn/src/asm/patches/mod.rs

@@ -57,3 +57,24 @@ impl<T, E: std::error::Error + Send + Sync + 'static> ErrWithPos<T> for Result<T
         }
     }
 }
+
+pub trait ErrSetFile<T> {
+    fn err_file(self, file: u32) -> Result<T, CrsnError>;
+}
+
+impl<T> ErrSetFile<T> for Result<T, CrsnError> {
+    /// Set file context in the error
+    fn err_file(self, file: u32) -> Result<T, CrsnError> {
+        match self {
+            Ok(v) => Ok(v),
+            Err(CrsnError::Parse(a, p)) => Err(CrsnError::Parse(a, p.with_file(file))),
+            Err(CrsnError::ParseOther(a, p)) => Err(CrsnError::ParseOther(a, p.with_file(file))),
+            Err(CrsnError::Asm(a, p)) => Err(CrsnError::Asm(a, p.with_file(file))),
+            Err(CrsnError::Sexp(mut se)) => {
+                se.pos.file = file;
+                Err(CrsnError::Sexp(se))
+            },
+            Err(other @ CrsnError::IOError(_)) => Err(other),
+        }
+    }
+}

crsn/src/asm/read_file.rs

@@ -4,7 +4,7 @@ use std::io::Read;
 use std::path::Path;
 
 /// Read a file to string
-pub fn read_file<P: AsRef<Path>>(path: P) -> io::Result<String> {
+pub fn read_file(path: impl AsRef<Path>) -> io::Result<String> {
     let path = path.as_ref();
     let mut file = File::open(path)?;
 

crsn/src/runtime/program.rs

@@ -10,6 +10,7 @@ use crate::asm::instr::op::OpKind;
 use crate::builtin::defs::{Barrier, BuiltinOp};
 use crate::module::CrsnExtension;
 use crate::runtime::fault::Fault;
+use std::path::PathBuf;
 
 #[derive(Debug)]
 pub struct Program {
@@ -20,16 +21,23 @@ pub struct Program {
     /// Barriers from-to (inclusive).
     /// Standalone barriers have both addresses the same.
     barriers: Vec<(Addr, Addr)>,
+    /// This is used at runtime to better report error locations when included files are used
+    pub(crate) file_names: Vec<PathBuf>,
 }
 
 impl Program {
-    pub fn new(ops: Vec<Op>, extensions: Arc<Vec<Box<dyn CrsnExtension>>>) -> Result<Arc<Self>, CrsnError> {
+    pub fn new(
+        ops: Vec<Op>,
+        extensions: Arc<Vec<Box<dyn CrsnExtension>>>,
+        file_names: Vec<PathBuf>,
+    ) -> Result<Arc<Self>, CrsnError> {
         let mut p = Self {
             ops,
             extensions,
             routines: Default::default(),
             far_labels: Default::default(),
             barriers: Default::default(),
+            file_names,
         };
         p.scan()?;
         Ok(Arc::new(p))

crsn/src/runtime/run_thread.rs

@@ -279,12 +279,14 @@ impl RunThread {
                 debug!("Thread ended.");
             }
             e => {
+                eprintln!("*** Program failed: {} ***", e);
                 if let Some(instr) = self.info.program.ops.get(orig_pc.0 as usize) {
-                    error!("Fault at {}: {}", instr.pos(), e);
+                    eprintln!("Source location: {}, line {}, column {}", self.info.program.file_names[instr.pos().file as usize].display(), instr.pos().line, instr.pos().column);
                 } else {
-                    error!("Fault at PC {}: {}", orig_pc, e);
+                    eprintln!("Instruction address: {}", orig_pc);
                 }
-                warn!("Core dump: {:?}", self.state);
+
+                debug!("\nCore dump: {:?}", self.state);
             }
         }
     }

examples/include/main.csn

@@ -0,0 +1,18 @@
+(
+    (include other)
+    (def BAR (=add FOO 1000))
+
+    (cmp BAR 1123 (ne? (fault)))
+
+    (include utils/itoa.csn)
+    (include "utils/printnum")
+
+    (mkbf r0)
+    (call itoa r0 BAR)
+    (lds @cout @r0)
+    (del @r0)
+    (ld @cout '\n')
+
+    (call printnum BAR)
+    (ld @cout '\n')
+)

examples/include/other.csn

@@ -0,0 +1,3 @@
+(
+    (def FOO 123)
+)

examples/include/utils/itoa.csn

@@ -0,0 +1,14 @@
+(
+    (proc itoa buf num
+        (ld r1 num)
+        (tst r1 (<0? (mul r1 -1)))
+        (:next)
+            (mod r0 r1 10)
+            (add r0 '0')
+            (bfrpush @buf r0)
+            (div r1 10 (z?
+                (tst num (<0? (bfrpush @buf '-')))
+                (ret)))
+            (j :next)
+    )
+)

examples/include/utils/printnum.csn

@@ -0,0 +1,17 @@
+(
+    (proc printnum num
+        (mkbf r15)
+        (ld r1 num)
+        (tst r1 (<0? (mul r1 -1)))
+        (:next)
+            (mod r0 r1 10)
+            (add r0 '0')
+            (bfrpush @r15 r0)
+            (div r1 10 (z?
+                (tst num (<0? (bfrpush @r15 '-')))
+                (lds @cout @r15)
+                (del @r15)
+                (ret)))
+            (j :next)
+    )
+)

launcher/src/main.rs

@@ -17,7 +17,6 @@ use crsn_screen::ScreenOps;
 use crsn_stdio::StdioOps;
 use crsn_buf::BufOps;
 
-mod read_file;
 mod serde_duration_millis;
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -167,11 +166,9 @@ fn main() -> anyhow::Result<()> {
 
     debug!("Loading {}", config.program_file);
 
-    let source = read_file::read_file(&config.program_file)?;
-
     let uniq = CrsnUniq::new();
 
-    let parsed = crsn::asm::assemble(&source, &uniq, vec![
+    let parsed = crsn::asm::assemble(&config.program_file, &uniq, vec![
         ArithOps::new(),
         BufOps::new(),
         ScreenOps::new(),