Merge pull request #410 from bedatadriven/srcrefs

addition "srcref" attribute to parsed expressions returned by parse()

core/src/main/R/base/srcfile.R

@@ -1,5 +1,7 @@
 #  File src/library/base/R/srcfile.R
-#  Part of the R package, http://www.R-project.org
+#  Part of the R package, https://www.R-project.org
+#
+#  Copyright (C) 1995-2017 The R Core Team
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -12,7 +14,7 @@
 #  GNU General Public License for more details.
 #
 #  A copy of the GNU General Public License is available at
-#  http://www.r-project.org/Licenses/
+#  https://www.R-project.org/Licenses/
 
 # a srcfile is a file with a timestamp
 
@@ -110,7 +112,7 @@ srcfilecopy <- function (filename, lines, timestamp = Sys.time(), isFile = FALSE
     e <- new.env(parent=emptyenv())
 
     # Remove embedded newlines
-    if (any(grepl("\n", lines, fixed=TRUE)))
+    if (any(grepl("\n", lines, fixed = TRUE, useBytes = TRUE)))
 	lines <- unlist(strsplit(sub("$", "\n", as.character(lines)), "\n"))
 
     e$filename <- filename
@@ -180,7 +182,7 @@ getSrcLines <- function(srcfile, first, last) {
 	# Remove embedded newlines if we haven't done this already
 	if (is.null(srcfile$fixedNewlines)) {
 	    lines <- srcfile$lines
-    	    if (any(grepl("\n", lines, fixed=TRUE))) 
+    	    if (any(grepl("\n", lines, fixed = TRUE, useBytes = TRUE)))
 		srcfile$lines <- unlist(strsplit(sub("$", "\n", as.character(lines)), "\n"))
 	    srcfile$fixedNewlines <- TRUE
 	}
@@ -210,24 +212,31 @@ srcref <- function(srcfile, lloc) {
     structure(as.integer(lloc), srcfile=srcfile, class="srcref")
 }
 
-as.character.srcref <- function(x, useSource = TRUE, ...)
+as.character.srcref <- function(x, useSource = TRUE, to = x, ...)
 {
     srcfile <- attr(x, "srcfile")
+    if (!missing(to)) {
+        if (!identical(srcfile, attr(to, "srcfile")))
+    	    stop("'x' and 'to' must refer to same file")
+    	x[c(3L, 4L, 6L, 8L)] <- to[c(3L, 4L, 6L, 8L)]
+    }
     if (!is.null(srcfile) && !inherits(srcfile, "srcfile")) {
-       cat("forcing class on") ## debug
-	print(str(srcfile))
-       class(srcfile) <- c("srcfilealias", "srcfile")
+        cat("forcing class on") ## debug
+        print(utils::str(srcfile))
+        class(srcfile) <- c("srcfilealias", "srcfile")
     }
+
     if (useSource) {
     	if (inherits(srcfile, "srcfilecopy") || inherits(srcfile, "srcfilealias"))
     	    lines <- try(getSrcLines(srcfile, x[7L], x[8L]), TRUE)
     	else
  	    lines <- try(getSrcLines(srcfile, x[1L], x[3L]), TRUE)
     }
     if (!useSource || inherits(lines, "try-error"))
-    	lines <- paste("<srcref: file \"", srcfile$filename, "\" chars ",
-                       x[1L],":",x[5L], " to ",x[3L],":",x[6L], ">", sep="")
-    else {
+    	lines <- paste0("<srcref: file \"", srcfile$filename, "\" chars ",
+                        x[1L], ":", x[5L], " to ",
+                        x[3L], ":", x[6L], ">")
+    else if (length(lines)) {
     	enc <- Encoding(lines)
     	Encoding(lines) <- "latin1"  # so byte counting works
         if (length(lines) < x[3L] - x[1L] + 1L)

core/src/main/java/org/renjin/parser/ParseState.java

@@ -35,7 +35,7 @@ public class ParseState {
   /**
    * Whether to attach srcrefs to objects as they are parsed
    */
-  boolean keepSrcRefs = true;
+  boolean keepSrcRefs = false;
 
   /**
    * The srcfile object currently being parsed
@@ -68,4 +68,8 @@ public SEXP getSrcFile() {
   public void setSrcFile(SEXP srcFile) { 
     this.srcFile = srcFile; 
   }
+
+  public void setKeepSrcRefs(boolean keepSrcRefs) {
+    this.keepSrcRefs = keepSrcRefs;
+  }
 }

core/src/main/java/org/renjin/parser/Position.java

@@ -26,9 +26,9 @@
  */
 public class Position {
 
-  public int line;
-  public int column;
-  public int charIndex;
+  private int line;
+  private int column;
+  private int charIndex;
 
   public Position() {
   }
@@ -41,7 +41,7 @@ public Position(int line, int column, int charIndex) {
 
   @Override
   public Position clone() {
-    return new Position(line, column, charIndex);
+    return new Position(getLine(), getColumn(), getCharIndex());
   }
 
   @Override
@@ -55,13 +55,13 @@ public boolean equals(Object o) {
 
     Position position = (Position) o;
 
-    if (charIndex != position.charIndex) {
+    if (getCharIndex() != position.getCharIndex()) {
       return false;
     }
-    if (column != position.column) {
+    if (getColumn() != position.getColumn()) {
       return false;
     }
-    if (line != position.line) {
+    if (getLine() != position.getLine()) {
       return false;
     }
 
@@ -70,14 +70,27 @@ public boolean equals(Object o) {
 
   @Override
   public int hashCode() {
-    int result = line;
-    result = 31 * result + column;
-    result = 31 * result + charIndex;
+    int result = getLine();
+    result = 31 * result + getColumn();
+    result = 31 * result + getCharIndex();
     return result;
   }
 
   @Override
   public String toString() {
-    return "line " + (line + 1) + " char " + column;
+    return "line " + (getLine() + 1) + " byte " + (getCharIndex() + 1) + " col " + (getColumn() + 1);
   }
+
+  public int getLine() {
+    return line;
+  }
+
+  public int getColumn() {
+    return column;
+  }
+
+  public int getCharIndex() {
+    return charIndex;
+  }
+
 }

core/src/main/java/org/renjin/parser/RLexer.java

@@ -104,11 +104,11 @@ public RLexer(ParseOptions options, ParseState state, Reader reader) {
 
 
   public Position getStartPos() {
-    return tokenBegin.clone();
+    return tokenBegin;
   }
 
   public Position getEndPos() {
-    return tokenEnd.clone();
+    return tokenEnd;
   }
 
   public SEXP getLVal() {
@@ -191,7 +191,7 @@ public int yylex() {
           } else {
             contextStack.ifPop();
             savedToken = token;
-            savedTokenPos = tokenBegin.clone();
+            savedTokenPos = tokenBegin;
             savedLVal = yylval;
             setlastloc();
             return '\n';
@@ -355,9 +355,7 @@ private int consumeNextToken() {
       c = skipComment();
     }
 
-    tokenBegin.line = reader.getLineNumber();
-    tokenBegin.column = reader.getColumnNumber();
-    tokenBegin.charIndex = reader.getCharacterIndex();
+    tokenBegin = reader.getPosition();
 
     if (c == R_EOF) {
       return END_OF_INPUT;
@@ -593,9 +591,7 @@ public String getErrorMessage() {
  */
 
   void setlastloc() {
-    tokenEnd.line = reader.getLineNumber();
-    tokenEnd.column = reader.getColumnNumber();
-    tokenEnd.charIndex = reader.getCharacterIndex();
+    tokenEnd = reader.getPosition();
   }
 
 

core/src/main/java/org/renjin/parser/RLexerReader.java

@@ -36,16 +36,21 @@ public class RLexerReader {
   private int prevpos = 0;
   private int prevlines[] = new int[PUSHBACK_BUFSIZE];
   private int prevcols[] = new int[PUSHBACK_BUFSIZE];
+  private int prevbytes[] = new int[PUSHBACK_BUFSIZE];
 
-  private int columnNumber = 0;
-  private int lineNumber = 1;
+  private int lineNumber = 0;
   private int charIndex = -1;
+  private int columnNumber = -1;
 
   public RLexerReader(Reader reader) {
     super();
     this.reader = new PushbackReader(reader);
   }
 
+  public Position getPosition() {
+    return new Position(lineNumber, columnNumber, charIndex);
+  }
+
   public int read() throws IOException {
     int c;
 
@@ -62,30 +67,32 @@ public int read() throws IOException {
     prevpos = (prevpos + 1) % PUSHBACK_BUFSIZE;
     prevcols[prevpos] = columnNumber;
     prevlines[prevpos] = lineNumber;
+    prevbytes[prevpos] = charIndex;
 
-    if (c == '\n') {
-      lineNumber += 1;
-      columnNumber = 0;
+    if(c == '\n') {
+      lineNumber++;
+      columnNumber = -1;
+      charIndex = -1;
     } else {
       columnNumber++;
+      charIndex++;
     }
 
     if (c == '\t') { 
       columnNumber = ((columnNumber + 7) & ~7);
     }
-    charIndex++;
 
     return c;
   }
 
   public int unread(int c) {
     lineNumber = prevlines[prevpos];
     columnNumber = prevcols[prevpos];
+    charIndex = prevbytes[prevpos];
     prevpos = (prevpos + PUSHBACK_BUFSIZE - 1) % PUSHBACK_BUFSIZE;
 
     // if ( KeepSource && GenerateCode && FunctionLevel > 0 )
     // SourcePtr--;
-    charIndex--;
     //R_ParseContext[R_ParseContextLast] = '\0';
     /* precaution as to how % is implemented for < 0 numbers */
     //  R_ParseContextLast = (R_ParseContextLast + PARSE_CONTEXT_SIZE -1) % PARSE_CONTEXT_SIZE;
@@ -96,14 +103,24 @@ public int unread(int c) {
     return c;
   }
 
+  /**
+   * @return The zero-based line number of the character we just read. The newline character
+   * is considered to be the last character of the line it terminates.
+   */
   public int getLineNumber() {
     return lineNumber;
   }
 
+  /**
+   * @return The zero-based index of the character we just read within the current line.
+   */
   public int getColumnNumber() {
     return columnNumber;
   }
 
+  /**
+   * @return The zero-based index of the character we just read within the whole character stream.
+   */
   public int getCharacterIndex() {
     return charIndex;
   }