Skip to content

[WIP] Add support for '''- delimited strings (dedented string literals) in Scala 3 #702

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: idea252.x
Choose a base branch
from
Draft

[WIP] Add support for '''- delimited strings (dedented string literals) in Scala 3 #702

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 6 additions & 10 deletions .idea/kotlinc.xml
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions scala/scala-impl/src/org/jetbrains/plugins/scala/highlighter/ScalaSyntaxHighlighter.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,10 +157,12 @@ object ScalaSyntaxHighlighter {
private val tSTRINGS = TokenSet.create(
tSTRING,
tMULTILINE_STRING,
tDEDENTED_STRING,
tWRONG_STRING,
tCHAR,
tSYMBOL,
tINTERPOLATED_MULTILINE_STRING,
tINTERPOLATED_DEDENTED_STRING,
tINTERPOLATED_STRING,
tINTERPOLATED_STRING_ID,
tINTERPOLATED_STRING_END
Expand Down
9 changes: 8 additions & 1 deletion scala/scala-impl/src/org/jetbrains/plugins/scala/lang/lexer/ScalaTokenTypes.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,11 @@ public interface ScalaTokenTypes {
IElementType tINTERPOLATED_MULTILINE_RAW_STRING = new ScalaTokenType("interpolated multiline raw string");
IElementType tINTERPOLATED_RAW_STRING = new ScalaTokenType("interpolated raw string");

// Dedented string literals (Scala 3)
IElementType tDEDENTED_STRING = new ScalaTokenType("dedented string");
IElementType tINTERPOLATED_DEDENTED_STRING = new ScalaTokenType("interpolated dedented string");
IElementType tINTERPOLATED_DEDENTED_RAW_STRING = new ScalaTokenType("interpolated dedented raw string");

IElementType tCHAR = new ScalaTokenType("Character");
IElementType tSYMBOL = new ScalaTokenType("Symbol");

Expand Down Expand Up @@ -267,7 +272,9 @@ public interface ScalaTokenTypes {
tMULTILINE_STRING,
tINTERPOLATED_STRING,
tINTERPOLATED_MULTILINE_STRING,
tINTERPOLATED_STRING_END
tINTERPOLATED_STRING_END,
tDEDENTED_STRING,
tINTERPOLATED_DEDENTED_STRING
);

TokenSet VAL_VAR_TOKEN_SET = TokenSet.create(kVAL, kVAR);
Expand Down
140 changes: 136 additions & 4 deletions scala/scala-impl/src/org/jetbrains/plugins/scala/lang/lexer/core/_ScalaCoreLexer.flex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,17 @@ import static org.jetbrains.plugins.scala.lang.lexer.ScalaTokenTypes.*;
}
}

private static class DedentedLevel extends InterpolatedStringLevel {
public final int delimiterLength;
public DedentedLevel(CharSequence interpolator, int delimiterLength) {
super(interpolator);
this.delimiterLength = delimiterLength;
}
public int getState() {
return INSIDE_DEDENTED_INTERPOLATED_STRING;
}
}

private boolean isScala3;

//
Expand All @@ -94,6 +105,7 @@ import static org.jetbrains.plugins.scala.lang.lexer.ScalaTokenTypes.*;
//to get id after $ in interpolated String
private boolean haveIdInString = false;
private boolean haveIdInMultilineString = false;
private boolean haveIdInDedentedString = false;
// Currently opened interpolated Strings. Each int represents the number of the opened left structural braces in the String
private Stack<InterpolatedStringLevel> nestedString = new Stack<>();
private CharSequence lastSeenInterpolator = null;
Expand All @@ -105,16 +117,49 @@ import static org.jetbrains.plugins.scala.lang.lexer.ScalaTokenTypes.*;
public void resetCustom() {
haveIdInString = false;
haveIdInMultilineString = false;
haveIdInDedentedString = false;
nestedString.clear();
lastSeenInterpolator = null;
}

private int countLeadingQuotes(CharSequence text) {
int count = 0;
for (int i = 0; i < text.length() && text.charAt(i) == '\''; i++) {
count++;
}
return count;
}

private boolean endsWithQuotes(CharSequence text, int expectedCount) {
if (text.length() < expectedCount) return false;
int count = 0;
for (int i = text.length() - 1; i >= 0 && text.charAt(i) == '\''; i--) {
count++;
}
return count >= expectedCount;
}

private boolean isValidDedentedString(CharSequence text) {
int leadingQuotes = countLeadingQuotes(text);
if (leadingQuotes < 3) return false; // Must have at least 3 quotes

// Find the ending quotes
int trailingQuotes = 0;
for (int i = text.length() - 1; i >= 0 && text.charAt(i) == '\''; i--) {
trailingQuotes++;
}

return leadingQuotes == trailingQuotes;
}

public boolean isInterpolatedStringState() {
return isInsideInterpolatedString() ||
haveIdInString ||
haveIdInMultilineString ||
haveIdInDedentedString ||
yystate() == INSIDE_INTERPOLATED_STRING ||
yystate() == INSIDE_MULTI_LINE_INTERPOLATED_STRING;
yystate() == INSIDE_MULTI_LINE_INTERPOLATED_STRING ||
yystate() == INSIDE_DEDENTED_INTERPOLATED_STRING;
}

private boolean shouldProcessBracesForInterpolated() {
Expand Down Expand Up @@ -145,6 +190,9 @@ import static org.jetbrains.plugins.scala.lang.lexer.ScalaTokenTypes.*;
} else if (haveIdInMultilineString) {
haveIdInMultilineString = false;
yybegin(INSIDE_MULTI_LINE_INTERPOLATED_STRING);
} else if (haveIdInDedentedString) {
haveIdInDedentedString = false;
yybegin(INSIDE_DEDENTED_INTERPOLATED_STRING);
}
}

Expand All @@ -158,6 +206,8 @@ import static org.jetbrains.plugins.scala.lang.lexer.ScalaTokenTypes.*;
typeAdjusted = tINTERPOLATED_RAW_STRING;
else if (type == tINTERPOLATED_MULTILINE_STRING && isInsideRawInterpolator())
typeAdjusted = tINTERPOLATED_MULTILINE_RAW_STRING;
else if (type == tINTERPOLATED_DEDENTED_STRING && isInsideRawInterpolator())
typeAdjusted = tINTERPOLATED_DEDENTED_RAW_STRING;
else
typeAdjusted = type;

Expand All @@ -166,6 +216,11 @@ import static org.jetbrains.plugins.scala.lang.lexer.ScalaTokenTypes.*;

@NotNull
private IElementType processDollarInsideString(boolean isInsideMultiline) {
return processDollarInsideString(isInsideMultiline, false);
}

@NotNull
private IElementType processDollarInsideString(boolean isInsideMultiline, boolean isInsideDedented) {
final IElementType token;

// TODO: remove this chech, this should always be false, cause $$ is handled by INTERPOLATED_STRING_ESCAPE pattern earlier
Expand All @@ -175,7 +230,9 @@ import static org.jetbrains.plugins.scala.lang.lexer.ScalaTokenTypes.*;
token = tINTERPOLATED_STRING_ESCAPE;
}
else {
if (isInsideMultiline) {
if (isInsideDedented) {
haveIdInDedentedString = true;
} else if (isInsideMultiline) {
haveIdInMultilineString = true;
} else {
haveIdInString = true;
Expand Down Expand Up @@ -271,12 +328,19 @@ hexDigit = [0-9A-Fa-f]
CHAR_ESCAPE_SEQUENCE = \\[^\r\n]
UNICODE_ESCAPE = \\u+ {hexDigit}{hexDigit}{hexDigit}{hexDigit} // Scala supports 1. multiple `u` chars after `\` 2. even \u000A ('\n') and \u000D (unlike Java)
ESCAPE_SEQUENCE = {UNICODE_ESCAPE} | {CHAR_ESCAPE_SEQUENCE}
CHARACTER_LITERAL = "'"([^\\\'\r\n]|{ESCAPE_SEQUENCE}|{OCTAL_ESCAPE_LITERAL})("'"|\\) | \'\\u000A\' | "'''" // TODO: \'\\u000A\' is redundunt, remove
CHARACTER_LITERAL = "'"([^\\\'\r\n]|{ESCAPE_SEQUENCE}|{OCTAL_ESCAPE_LITERAL})("'"|\\) | \'\\u000A\' // TODO: \'\\u000A\' is redundunt, remove

STRING_BEGIN = \"([^\\\"\r\n]|{CHAR_ESCAPE_SEQUENCE})*
STRING_LITERAL={STRING_BEGIN} \"
MULTI_LINE_STRING = \"\"\" ( (\"(\")?)? [^\"] )* \"\"\" (\")* // Multi-line string

// Dedented string literals (Scala 3) - modeled after MULTI_LINE_STRING pattern
DEDENTED_STRING_3 = \'\'\' ( (\'(\')?)? [^\'] )* \'\'\' (\')*
DEDENTED_STRING_4 = \'\'\'\' ( (\'(\'\'?)?)? [^\'] )* \'\'\'\' (\')*
DEDENTED_STRING_5 = \'\'\'\'\' ( (\'(\'\'\'?)?)? [^\'] )* \'\'\'\'\' (\')*
DEDENTED_STRING_6 = \'\'\'\'\'\' ( (\'(\'\'\'\'?)?)? [^\'] )* \'\'\'\'\'\' (\')*
DEDENTED_STRING = {DEDENTED_STRING_6} | {DEDENTED_STRING_5} | {DEDENTED_STRING_4} | {DEDENTED_STRING_3}

////////String Interpolation////////
INTERPOLATED_STRING_ID = {varid}

Expand All @@ -287,6 +351,9 @@ INTERPOLATED_STRING_PART_NOT_ESCAPED = [^\\\"\r\n\$]
INTERPOLATED_MULTI_LINE_STRING_BEGIN = \"\"\"{INTERPOLATED_MULTI_LINE_STRING_PART}*
INTERPOLATED_MULTI_LINE_STRING_PART = ((\"(\")?)? [^\"\$])

INTERPOLATED_DEDENTED_STRING_BEGIN = \'\'\'+ {INTERPOLATED_DEDENTED_STRING_PART}*
INTERPOLATED_DEDENTED_STRING_PART = [^\'\$] | \$ [^{] | \'[^\']+

// TODO: rename, it's missleading
INTERPOLATED_STRING_ESCAPE = "$$"
//INTERPOLATED_STRING_VARIABLE = "$"({identifier})
Expand Down Expand Up @@ -324,6 +391,7 @@ XML_BEGIN = "<" ("_" | [:jletter:]) | "<!--" | "<?" ("_" | [:jletter:]) | "<![CD
%xstate WAIT_FOR_INTERPOLATED_STRING
%xstate INSIDE_INTERPOLATED_STRING
%xstate INSIDE_MULTI_LINE_INTERPOLATED_STRING
%xstate INSIDE_DEDENTED_INTERPOLATED_STRING
%xstate INJ_COMMON_STATE

%%
Expand All @@ -344,7 +412,7 @@ XML_BEGIN = "<" ("_" | [:jletter:]) | "<!--" | "<?" ("_" | [:jletter:]) | "<![CD
{END_OF_LINE_COMMENT} { return process(tLINE_COMMENT); }


{INTERPOLATED_STRING_ID} / ({INTERPOLATED_STRING_BEGIN} | {INTERPOLATED_MULTI_LINE_STRING_BEGIN}) {
{INTERPOLATED_STRING_ID} / ({INTERPOLATED_STRING_BEGIN} | {INTERPOLATED_MULTI_LINE_STRING_BEGIN} | {INTERPOLATED_DEDENTED_STRING_BEGIN}) {
yybegin(WAIT_FOR_INTERPOLATED_STRING);
// TODO: remove this check: looks like it's a dead code,
// yytext() should only return text that is matched by INTERPOLATED_STRING_ID, which can't end with \"\"
Expand All @@ -367,6 +435,13 @@ XML_BEGIN = "<" ("_" | [:jletter:]) | "<!--" | "<?" ("_" | [:jletter:]) | "<![CD
nestedString.push(new MultilineLevel(lastSeenInterpolator));
return process(tINTERPOLATED_MULTILINE_STRING);
}

{INTERPOLATED_DEDENTED_STRING_BEGIN} {
yybegin(INSIDE_DEDENTED_INTERPOLATED_STRING);
int delimiterLength = countLeadingQuotes(yytext());
nestedString.push(new DedentedLevel(lastSeenInterpolator, delimiterLength));
return process(tINTERPOLATED_DEDENTED_STRING);
}
}

<INJ_COMMON_STATE> {identifier} {
Expand Down Expand Up @@ -470,6 +545,61 @@ XML_BEGIN = "<" ("_" | [:jletter:]) | "<!--" | "<?" ("_" | [:jletter:]) | "<![CD
}
}

<INSIDE_DEDENTED_INTERPOLATED_STRING> {
{INTERPOLATED_STRING_ESCAPE} {
return process(tINTERPOLATED_STRING_ESCAPE);
}

(\'\') / "$" {
return process(tINTERPOLATED_DEDENTED_STRING);
}

{INTERPOLATED_DEDENTED_STRING_PART}+ {
return process(tINTERPOLATED_DEDENTED_STRING);
}

"$"{identifier} {
return processDollarInsideString(false, true);
}

\'\'\'+ (\')+ {
yypushback(yylength() - 1);
return process(tINTERPOLATED_DEDENTED_STRING);
}

\'\'\'+ {
// Check if this ends the dedented string with matching delimiter length
if (!nestedString.isEmpty() && nestedString.peek() instanceof DedentedLevel) {
DedentedLevel level = (DedentedLevel) nestedString.peek();
int quoteCount = yylength();
if (quoteCount == level.delimiterLength) {
return processOutsideString();
} else if (quoteCount < level.delimiterLength) {
// Not enough quotes to close, treat as content
return process(tINTERPOLATED_DEDENTED_STRING);
} else {
// Too many quotes, pushback the excess
yypushback(quoteCount - level.delimiterLength);
return processOutsideString();
}
}
return processOutsideString();
}

"$" / "{" {
yybegin(COMMON_STATE);
return process(tINTERPOLATED_STRING_INJECTION);
}

\' / [^\'] {
return process(tINTERPOLATED_DEDENTED_STRING);
}

[^] {
return process(tWRONG_STRING);
}
}


"/**" ("*"? [^\/])* "*/" { //for comments in interpolated strings
return process(ScalaDocElementTypes.SCALA_DOC_COMMENT);
Expand All @@ -486,6 +616,8 @@ XML_BEGIN = "<" ("_" | [:jletter:]) | "<!--" | "<?" ("_" | [:jletter:]) | "<![CD
// TODO: incomplete strings should be handled the same way with interpolated strings
// what can be parsed should be parsed as tSTRING,
// tWRONG_LINE_BREAK_IN_STRING error token should be added at unexpected new line should
{DEDENTED_STRING} { if (isScala3) return process(tDEDENTED_STRING); else return process(tIDENTIFIER); }

{WRONG_STRING} { return process(tWRONG_STRING); }


Expand Down
Loading
Loading