Skip to content

Instantly share code, notes, and snippets.

@headius
Last active Dec 20, 2015
Embed
What would you like to do?
diff --git a/core/src/main/java/org/jruby/RubyString.java b/core/src/main/java/org/jruby/RubyString.java
index 7551cec..b1def3d 100644
--- a/core/src/main/java/org/jruby/RubyString.java
+++ b/core/src/main/java/org/jruby/RubyString.java
@@ -7376,7 +7376,7 @@ public class RubyString extends RubyObject implements EncodingCapable, MarshalEn
*
*/
private RubySymbol to_sym() {
- RubySymbol specialCaseIntern = checkSpecialCasesIntern(value);
+ RubySymbol specialCaseIntern = checkSpecialCasesIntern(Helpers.symbolBytesToString(value));
if (specialCaseIntern != null) return specialCaseIntern;
RubySymbol symbol = getRuntime().getSymbolTable().getSymbol(value);
@@ -7384,13 +7384,14 @@ public class RubyString extends RubyObject implements EncodingCapable, MarshalEn
return symbol;
}
- private RubySymbol checkSpecialCasesIntern(ByteList value) {
- String[][] opTable = getRuntime().is1_8() ? opTable18 : opTable19;
+ private RubySymbol checkSpecialCasesIntern(String value) {
+ Ruby runtime = getRuntime();
+ String[][] opTable = runtime.is1_8() ? opTable18 : opTable19;
for (int i = 0; i < opTable.length; i++) {
String op = opTable[i][1];
- if (value.toString().equals(op)) {
- return getRuntime().getSymbolTable().getSymbol(opTable[i][0]);
+ if (value.equals(op)) {
+ return runtime.getSymbolTable().getSymbol(opTable[i][0]);
}
}
diff --git a/core/src/main/java/org/jruby/RubyStruct.java b/core/src/main/java/org/jruby/RubyStruct.java
index 97ec24d..3189790 100644
--- a/core/src/main/java/org/jruby/RubyStruct.java
+++ b/core/src/main/java/org/jruby/RubyStruct.java
@@ -169,7 +169,7 @@ public class RubyStruct extends RubyObject {
if (args.length > 0) {
IRubyObject firstArgAsString = args[0].checkStringType();
if (!firstArgAsString.isNil()) {
- name = ((RubyString)firstArgAsString).getByteList().toString();
+ name = Helpers.decodeByteList(runtime, ((RubyString)firstArgAsString).getByteList());
} else if (args[0].isNil()) {
nilName = true;
}
diff --git a/core/src/main/java/org/jruby/RubySymbol.java b/core/src/main/java/org/jruby/RubySymbol.java
index 40c2233..126a03e 100644
--- a/core/src/main/java/org/jruby/RubySymbol.java
+++ b/core/src/main/java/org/jruby/RubySymbol.java
@@ -47,6 +47,7 @@ import java.util.concurrent.locks.ReentrantLock;
import org.jcodings.Encoding;
import org.jcodings.specific.USASCIIEncoding;
+import org.jcodings.specific.UTF8Encoding;
import org.jruby.anno.JRubyClass;
import org.jruby.anno.JRubyMethod;
import org.jruby.ast.util.ArgsUtil;
@@ -57,6 +58,7 @@ import org.jruby.runtime.BlockBody;
import org.jruby.runtime.CallSite;
import org.jruby.runtime.ClassIndex;
import org.jruby.runtime.ContextAwareBlockBody;
+import org.jruby.runtime.Helpers;
import org.jruby.runtime.ObjectAllocator;
import org.jruby.runtime.ThreadContext;
import org.jruby.runtime.builtin.IRubyObject;
@@ -661,7 +663,14 @@ public class RubySymbol extends RubyObject implements MarshalEncoding {
private static ByteList symbolBytesFromString(Ruby runtime, String internedSymbol) {
if (runtime.is1_9()) {
- return new ByteList(ByteList.plain(internedSymbol), USASCIIEncoding.INSTANCE, false);
+ byte[] encodedBytes = internedSymbol.getBytes(UTF8Encoding.INSTANCE.getCharset());
+ if (encodedBytes.length == internedSymbol.length()) {
+ // all ASCII; use US-ASCII
+ return new ByteList(encodedBytes, USASCIIEncoding.INSTANCE, false);
+ }
+
+ // non-ASCII bytes, use UTF-8
+ return new ByteList(encodedBytes, UTF8Encoding.INSTANCE, false);
} else {
return ByteList.create(internedSymbol);
}
@@ -719,7 +728,7 @@ public class RubySymbol extends RubyObject implements MarshalEncoding {
RubySymbol symbol = bytelistTable.get(bytes);
if (symbol != null) return symbol;
- String name = bytes.toString();
+ String name = Helpers.symbolBytesToString(bytes);
int hash = name.hashCode();
SymbolEntry[] table = symbolTable;
diff --git a/core/src/main/java/org/jruby/compiler/ASTCompiler.java b/core/src/main/java/org/jruby/compiler/ASTCompiler.java
index 6f3d255..d04fac8 100644
--- a/core/src/main/java/org/jruby/compiler/ASTCompiler.java
+++ b/core/src/main/java/org/jruby/compiler/ASTCompiler.java
@@ -797,7 +797,7 @@ public class ASTCompiler {
if (arrayNode.get(0) instanceof SymbolNode) {
return ((SymbolNode)arrayNode.get(0)).getName();
} else if (arrayNode.get(0) instanceof StrNode) {
- return ((StrNode)arrayNode.get(0)).getValue().toString();
+ return Helpers.symbolBytesToString(((StrNode)arrayNode.get(0)).getValue());
}
}
}
diff --git a/core/src/main/java/org/jruby/parser/DefaultRubyParser.java b/core/src/main/java/org/jruby/parser/DefaultRubyParser.java
index a36c120..ae2fc24 100644
--- a/core/src/main/java/org/jruby/parser/DefaultRubyParser.java
+++ b/core/src/main/java/org/jruby/parser/DefaultRubyParser.java
@@ -130,6 +130,7 @@ import org.jruby.lexer.yacc.StrTerm;
import org.jruby.lexer.yacc.SyntaxException;
import org.jruby.lexer.yacc.SyntaxException.PID;
import org.jruby.lexer.yacc.Token;
+import org.jruby.runtime.Helpers;
import org.jruby.util.ByteList;
import org.jruby.util.RegexpOptions;
@@ -3065,7 +3066,7 @@ states[422] = new ParserState() {
if (((Node)yyVals[-1+yyTop]) instanceof DStrNode) {
yyVal = new DSymbolNode(((Token)yyVals[-2+yyTop]).getPosition(), ((DStrNode)yyVals[-1+yyTop]));
} else if (((Node)yyVals[-1+yyTop]) instanceof StrNode) {
- yyVal = new SymbolNode(((Token)yyVals[-2+yyTop]).getPosition(), ((StrNode)yyVals[-1+yyTop]).getValue().toString().intern());
+ yyVal = new SymbolNode(((Token)yyVals[-2+yyTop]).getPosition(), Helpers.symbolBytesToString(((StrNode)yyVals[-1+yyTop]).getValue()).intern());
} else {
yyVal = new DSymbolNode(((Token)yyVals[-2+yyTop]).getPosition());
((DSymbolNode)yyVal).add(((Node)yyVals[-1+yyTop]));
diff --git a/core/src/main/java/org/jruby/parser/ParserSupport.java b/core/src/main/java/org/jruby/parser/ParserSupport.java
index 0332d31..d507b8b 100644
--- a/core/src/main/java/org/jruby/parser/ParserSupport.java
+++ b/core/src/main/java/org/jruby/parser/ParserSupport.java
@@ -52,6 +52,7 @@ import org.jruby.lexer.yacc.SyntaxException;
import org.jruby.lexer.yacc.Token;
import org.jruby.lexer.yacc.SyntaxException.PID;
import org.jruby.runtime.DynamicScope;
+import org.jruby.runtime.Helpers;
import org.jruby.util.ByteList;
import org.jruby.util.IdUtil;
import org.jruby.util.RegexpOptions;
@@ -1147,7 +1148,7 @@ public class ParserSupport {
public Node asSymbol(ISourcePosition position, Node value) {
// FIXME: This might have an encoding issue since toString generally uses iso-8859-1
- if (value instanceof StrNode) return new SymbolNode(position, ((StrNode) value).getValue().toString());
+ if (value instanceof StrNode) return new SymbolNode(position, Helpers.symbolBytesToString(((StrNode) value).getValue()));
return new DSymbolNode(position, (DStrNode) value);
}
diff --git a/core/src/main/java/org/jruby/parser/Ruby19Parser.java b/core/src/main/java/org/jruby/parser/Ruby19Parser.java
index 8941e78..f21da20 100644
--- a/core/src/main/java/org/jruby/parser/Ruby19Parser.java
+++ b/core/src/main/java/org/jruby/parser/Ruby19Parser.java
@@ -116,6 +116,7 @@ import org.jruby.lexer.yacc.StrTerm;
import org.jruby.lexer.yacc.SyntaxException;
import org.jruby.lexer.yacc.SyntaxException.PID;
import org.jruby.lexer.yacc.Token;
+import org.jruby.runtime.Helpers;
import org.jruby.util.ByteList;
public class Ruby19Parser implements RubyParser {
@@ -3793,7 +3794,7 @@ states[452] = new ParserState() {
} else if (((Node)yyVals[-1+yyTop]) instanceof DStrNode) {
yyVal = new DSymbolNode(((Token)yyVals[-2+yyTop]).getPosition(), ((DStrNode)yyVals[-1+yyTop]));
} else if (((Node)yyVals[-1+yyTop]) instanceof StrNode) {
- yyVal = new SymbolNode(((Token)yyVals[-2+yyTop]).getPosition(), ((StrNode)yyVals[-1+yyTop]).getValue().toString().intern());
+ yyVal = new SymbolNode(((Token)yyVals[-2+yyTop]).getPosition(), Helpers.symbolBytesToString(((StrNode)yyVals[-1+yyTop]).getValue()).intern());
} else {
yyVal = new DSymbolNode(((Token)yyVals[-2+yyTop]).getPosition());
((DSymbolNode)yyVal).add(((Node)yyVals[-1+yyTop]));
diff --git a/core/src/main/java/org/jruby/parser/Ruby20Parser.java b/core/src/main/java/org/jruby/parser/Ruby20Parser.java
index 22170a9..d22e43e 100644
--- a/core/src/main/java/org/jruby/parser/Ruby20Parser.java
+++ b/core/src/main/java/org/jruby/parser/Ruby20Parser.java
@@ -116,6 +116,7 @@ import org.jruby.lexer.yacc.StrTerm;
import org.jruby.lexer.yacc.SyntaxException;
import org.jruby.lexer.yacc.SyntaxException.PID;
import org.jruby.lexer.yacc.Token;
+import org.jruby.runtime.Helpers;
import org.jruby.util.ByteList;
public class Ruby20Parser implements RubyParser {
@@ -2503,7 +2504,7 @@ states[476] = new ParserState() {
} else if (((Node)yyVals[-1+yyTop]) instanceof DStrNode) {
yyVal = new DSymbolNode(((Token)yyVals[-2+yyTop]).getPosition(), ((DStrNode)yyVals[-1+yyTop]));
} else if (((Node)yyVals[-1+yyTop]) instanceof StrNode) {
- yyVal = new SymbolNode(((Token)yyVals[-2+yyTop]).getPosition(), ((StrNode)yyVals[-1+yyTop]).getValue().toString().intern());
+ yyVal = new SymbolNode(((Token)yyVals[-2+yyTop]).getPosition(), Helpers.symbolBytesToString(((StrNode)yyVals[-1+yyTop]).getValue()).intern());
} else {
yyVal = new DSymbolNode(((Token)yyVals[-2+yyTop]).getPosition());
((DSymbolNode)yyVal).add(((Node)yyVals[-1+yyTop]));
diff --git a/core/src/main/java/org/jruby/runtime/marshal/MarshalStream.java b/core/src/main/java/org/jruby/runtime/marshal/MarshalStream.java
index 27d23c1..86b2eb0 100644
--- a/core/src/main/java/org/jruby/runtime/marshal/MarshalStream.java
+++ b/core/src/main/java/org/jruby/runtime/marshal/MarshalStream.java
@@ -481,8 +481,7 @@ public class MarshalStream extends FilterOutputStream {
public void writeString(String value) throws IOException {
writeInt(value.length());
- // FIXME: should preserve unicode?
- out.write(RubyString.stringToBytes(value));
+ out.write(value.getBytes(UTF8Encoding.INSTANCE.getCharset()));
}
public void writeString(ByteList value) throws IOException {
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment