feat: Tokens now store escaped and unescaped values

Author: quintesse

src/main/java/org/codejive/properties/PropertiesParser.java

@@ -21,37 +21,51 @@ public enum Type {
 
     public static class Token {
         final Type type;
+        final String raw;
         final String text;
 
-        Token(Type type, String text) {
+        Token(Type type, String raw) {
+            this(type, raw, null);
+        }
+
+        Token(Type type, String raw, String text) {
             this.type = type;
+            this.raw = raw;
             this.text = text;
         }
 
         public Type getType() {
             return type;
         }
 
+        public String getRaw() {
+            return raw;
+        }
+
         public String getText() {
-            return text;
+            return text != null ? text : raw;
         }
 
         @Override
         public boolean equals(Object o) {
             if (this == o) return true;
             if (!(o instanceof Token)) return false;
             Token token = (Token) o;
-            return text.equals(token.text);
+            return type == token.type && raw.equals(token.raw) && Objects.equals(text, token.text);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(text);
+            return Objects.hash(type, raw, text);
         }
 
         @Override
         public String toString() {
-            return "Token('" + type + ", " + text + "')";
+            if (text == null) {
+                return "Token(" + type + ", '" + raw + "')";
+            } else {
+                return "Token(" + type + ", '" + raw + "'" + ", '" + text + "')";
+            }
         }
     }
 
@@ -62,6 +76,7 @@ public String toString() {
     private int pch;
     StringBuilder chr;
     StringBuilder str;
+    boolean hasEscapes;
 
     public PropertiesParser(Reader rdr) throws IOException {
         this.rdr = rdr;
@@ -100,18 +115,22 @@ public Token nextToken() throws IOException {
                 nextState = null;
             }
             if (isValid.get()) {
-                str.append(chr);
                 nextChar();
             } else {
                 String text = (state == Type.VALUE || state == Type.COMMENT) ? trimmedString() : string();
-                Token token = new Token(state, text);
+                Token token = hasEscapes ? new Token(state, text, unescape(text)) :  new Token(state, text);
+                hasEscapes = false;
                 state = nextState;
                 return token;
             }
         }
     }
 
     private void nextChar() throws IOException {
+        str.append(chr);
+        if (chr.length() > 0 && chr.charAt(0) == '\\') {
+            hasEscapes = true;
+        }
         if (pch == -1) {
             ch = rdr.read();
         } else {
@@ -139,6 +158,41 @@ private void nextChar() throws IOException {
         }
     }
 
+    private static String unescape(String escape) {
+        StringBuilder txt = new StringBuilder();
+        for (int i = 0; i < escape.length(); i++) {
+            char ch = escape.charAt(i);
+            if (ch == '\\') {
+                ch = escape.charAt(++i);
+                switch (ch) {
+                    case 't':
+                        txt.append('\t');
+                        break;
+                    case 'f':
+                        txt.append('\f');
+                        break;
+                    case 'n':
+                        txt.append('\n');
+                        break;
+                    case 'r':
+                        txt.append('\r');
+                        break;
+                    case 'u':
+                        String num = escape.substring(i + 1, i + 5);
+                        txt.append((char) Integer.parseInt(num, 16));
+                        i += 4;
+                        break;
+                    default:
+                        txt.append(ch);
+                        break;
+                }
+            } else {
+                txt.append(ch);
+            }
+        }
+        return txt.toString();
+    }
+
     private void readEol(int cch) throws IOException {
         if (cch == '\n') {
             // If the next char is a \r we'll add it

src/test/java/org/codejive/properties/TestPropertiesParser.java

@@ -60,9 +60,9 @@ void testTokens() throws IOException {
                                 new Token(Type.WHITESPACE, "\n"),
                                 new Token(Type.KEY, "three"),
                                 new Token(Type.SEPARATOR, "="),
-                                new Token(Type.VALUE, "and escapes\\n\\t\\r\\f"),
+                                new Token(Type.VALUE, "and escapes\\n\\t\\r\\f", "and escapes\n\t\r\f"),
                                 new Token(Type.WHITESPACE, "\n  "),
-                                new Token(Type.KEY, "\\ with\\ spaces"),
+                                new Token(Type.KEY, "\\ with\\ spaces", " with spaces"),
                                 new Token(Type.SEPARATOR, "   =    "),
                                 new Token(Type.VALUE, "everywhere"),
                                 new Token(Type.WHITESPACE, "  \n"),
@@ -72,11 +72,11 @@ void testTokens() throws IOException {
                                 new Token(Type.WHITESPACE, "\n"),
                                 new Token(Type.KEY, "multiline"),
                                 new Token(Type.SEPARATOR, " = "),
-                                new Token(Type.VALUE, "one \\\n    two  \\\n\r\tthree"),
+                                new Token(Type.VALUE, "one \\\n    two  \\\n\r\tthree", "one \n    two  \n\r\tthree"),
                                 new Token(Type.WHITESPACE, "\n"),
                                 new Token(Type.KEY, "key.4"),
                                 new Token(Type.SEPARATOR, " = "),
-                                new Token(Type.VALUE, "\\u1234"),
+                                new Token(Type.VALUE, "\\u1234", "\u1234"),
                                 new Token(Type.WHITESPACE, "\n"),
                                 new Token(Type.COMMENT, "# final comment"))));
     }
@@ -85,7 +85,7 @@ void testTokens() throws IOException {
     void testStringify() throws IOException {
         StringReader rdr = new StringReader(props);
         Stream<Token> tokens = PropertiesParser.tokens(rdr);
-        String props2 = tokens.map(Token::getText).collect(Collectors.joining());
+        String props2 = tokens.map(Token::getRaw).collect(Collectors.joining());
         assertThat(props2, equalTo(props));
     }
 }