Enhanced version of dip1036 implementation (dlang#15715)

This implements the Enhanced Interpolated Expression Sequence proposal:

i"" or iq{} or q`` with a $(expression) in the middle are converted to a tuple of druntime types for future processing by library code.

changelog/dmd.ies.dd

@@ -0,0 +1,18 @@
+Add support for Interpolated Expression Sequences
+
+Interpolated Expression Sequences are a way to implement things like string interpolation in library code. Three forms of literals are added:
+
+```
+i"Content $(a + 4)"
+i`Content $(a + 4)`
+iq{Content $(a + 4)}
+```
+
+all provide the same thing: a tuple that can be passed to other functions, like `writeln` from `std.stdio` and `text` from `std.conv`:
+
+```
+int a = 6;
+writeln(i"Content $(a + 4)"); // prints "Content 10"
+```
+
+You can also pass them to other functions which understand the types in the new `core.interpolation` module. Numerous examples can be found documentation of that module or in this repository: https://github.com/adamdruppe/interpolation-examples/

compiler/src/dmd/astbase.d

@@ -4575,6 +4575,7 @@ struct ASTBase
             inout(SuperExp)     isSuperExp() { return op == EXP.super_ ? cast(typeof(return))this : null; }
             inout(NullExp)      isNullExp() { return op == EXP.null_ ? cast(typeof(return))this : null; }
             inout(StringExp)    isStringExp() { return op == EXP.string_ ? cast(typeof(return))this : null; }
+            inout(InterpExp)    isInterpExp() { return op == EXP.interpolated ? cast(typeof(return))this : null; }
             inout(TupleExp)     isTupleExp() { return op == EXP.tuple ? cast(typeof(return))this : null; }
             inout(ArrayLiteralExp) isArrayLiteralExp() { return op == EXP.arrayLiteral ? cast(typeof(return))this : null; }
             inout(AssocArrayLiteralExp) isAssocArrayLiteralExp() { return op == EXP.assocArrayLiteral ? cast(typeof(return))this : null; }
@@ -4907,6 +4908,25 @@ struct ASTBase
         }
     }
 
+    extern (C++) final class InterpExp : Expression
+    {
+        InterpolatedSet* interpolatedSet;
+        char postfix = 0;   // 'c', 'w', 'd'
+
+        extern (D) this(const ref Loc loc, InterpolatedSet* interpolatedSet, char postfix = 0)
+        {
+            super(loc, EXP.interpolated, __traits(classInstanceSize, InterpExp));
+            this.interpolatedSet = interpolatedSet;
+            this.postfix = postfix;
+        }
+
+        override void accept(Visitor v)
+        {
+            v.visit(this);
+        }
+    }
+
+
     extern (C++) final class StringExp : Expression
     {
         union

compiler/src/dmd/doc.d

@@ -5204,6 +5204,7 @@ void highlightCode2(Scope* sc, Dsymbols* a, ref OutBuffer buf, size_t offset)
             highlight = "$(D_COMMENT ";
             break;
         case TOK.string_:
+        case TOK.interpolated:
             highlight = "$(D_STRING ";
             break;
         default:
@@ -5216,7 +5217,7 @@ void highlightCode2(Scope* sc, Dsymbols* a, ref OutBuffer buf, size_t offset)
             res.writestring(highlight);
             size_t o = res.length;
             highlightCode3(sc, res, tok.ptr, lex.p);
-            if (tok.value == TOK.comment || tok.value == TOK.string_)
+            if (tok.value == TOK.comment || tok.value == TOK.string_ || tok.value == TOK.interpolated)
                 /* https://issues.dlang.org/show_bug.cgi?id=7656
                  * https://issues.dlang.org/show_bug.cgi?id=7715
                  * https://issues.dlang.org/show_bug.cgi?id=10519

compiler/src/dmd/expression.d

@@ -721,6 +721,7 @@ extern (C++) abstract class Expression : ASTNode
         inout(SuperExp)     isSuperExp() { return op == EXP.super_ ? cast(typeof(return))this : null; }
         inout(NullExp)      isNullExp() { return op == EXP.null_ ? cast(typeof(return))this : null; }
         inout(StringExp)    isStringExp() { return op == EXP.string_ ? cast(typeof(return))this : null; }
+        inout(InterpExp)    isInterpExp() { return op == EXP.interpolated ? cast(typeof(return))this : null; }
         inout(TupleExp)     isTupleExp() { return op == EXP.tuple ? cast(typeof(return))this : null; }
         inout(ArrayLiteralExp) isArrayLiteralExp() { return op == EXP.arrayLiteral ? cast(typeof(return))this : null; }
         inout(AssocArrayLiteralExp) isAssocArrayLiteralExp() { return op == EXP.assocArrayLiteral ? cast(typeof(return))this : null; }
@@ -1847,6 +1848,28 @@ extern (C++) final class StringExp : Expression
     }
 }
 
+extern (C++) final class InterpExp : Expression
+{
+    char postfix = NoPostfix;   // 'c', 'w', 'd'
+    OwnedBy ownedByCtfe = OwnedBy.code;
+    InterpolatedSet* interpolatedSet;
+
+    enum char NoPostfix = 0;
+
+    extern (D) this(const ref Loc loc, InterpolatedSet* set, char postfix = NoPostfix) scope
+    {
+        super(loc, EXP.interpolated);
+        this.interpolatedSet = set;
+        this.postfix = postfix;
+    }
+
+    override void accept(Visitor v)
+    {
+        v.visit(this);
+    }
+}
+
+
 /***********************************************************
  * A sequence of expressions
  *
@@ -5494,6 +5517,7 @@ private immutable ubyte[EXP.max+1] expSize = [
     EXP.preMinusMinus: __traits(classInstanceSize, PreExp),
     EXP.identifier: __traits(classInstanceSize, IdentifierExp),
     EXP.string_: __traits(classInstanceSize, StringExp),
+    EXP.interpolated: __traits(classInstanceSize, InterpExp),
     EXP.this_: __traits(classInstanceSize, ThisExp),
     EXP.super_: __traits(classInstanceSize, SuperExp),
     EXP.halt: __traits(classInstanceSize, HaltExp),

compiler/src/dmd/expression.h

@@ -38,6 +38,7 @@ class TemplateDeclaration;
 class ClassDeclaration;
 class OverloadSet;
 class StringExp;
+class InterpExp;
 class LoweredAssignExp;
 #ifdef IN_GCC
 typedef union tree_node Symbol;
@@ -129,6 +130,7 @@ class Expression : public ASTNode
     SuperExp* isSuperExp();
     NullExp* isNullExp();
     StringExp* isStringExp();
+    InterpExp* isInterpExp();
     TupleExp* isTupleExp();
     ArrayLiteralExp* isArrayLiteralExp();
     AssocArrayLiteralExp* isAssocArrayLiteralExp();
@@ -370,6 +372,16 @@ class StringExp final : public Expression
     void writeTo(void* dest, bool zero, int tyto = 0) const;
 };
 
+class InterpExp final : public Expression
+{
+public:
+    utf8_t postfix;   // 'c', 'w', 'd'
+    OwnedBy ownedByCtfe;
+    void* interpolatedSet;
+
+    void accept(Visitor* v) override { v->visit(this); }
+};
+
 // Tuple
 
 class TupleExp final : public Expression

compiler/src/dmd/expressionsem.d

@@ -4145,6 +4145,84 @@ private extern (C++) final class ExpressionSemanticVisitor : Visitor
         result = e;
     }
 
+    override void visit(InterpExp e)
+    {
+        // the lexer breaks up into an odd/even array of literals and expression code
+        // we need to turn that into:
+        /+
+            tuple(
+                .object.imported!"core.interpolation".InterpolationHeader(),
+                ...
+                .object.imported!"core.interpolation".InterpolationFooter()
+            )
+
+            There the ... loops through them all, making the even ones
+                .object.imported!"core.interpolation".InterpolatedLiteral!str()
+            and making the odd ones
+                .object.imported!"core.interpolation".InterpolatedExpression!str(),
+                the code represented by str
+
+            Empty string literals are skipped as they provide no additional information.
+        +/
+
+        if (e.postfix)
+            error(e.loc, "String postfixes on interpolated expression sequences are not allowed.");
+
+        Expression makeNonTemplateItem(Identifier which) {
+            Expression id = new IdentifierExp(e.loc, Id.empty);
+            id = new DotIdExp(e.loc, id, Id.object);
+            auto moduleNameArgs = new Objects();
+            moduleNameArgs.push(new StringExp(e.loc, "core.interpolation"));
+            id = new DotTemplateInstanceExp(e.loc, id, Id.imported, moduleNameArgs);
+            id = new DotIdExp(e.loc, id, which);
+            id = new CallExp(e.loc, id, new Expressions());
+            return id;
+        }
+
+        Expression makeTemplateItem(Identifier which, string arg) {
+            Expression id = new IdentifierExp(e.loc, Id.empty);
+            id = new DotIdExp(e.loc, id, Id.object);
+            auto moduleNameArgs = new Objects();
+            moduleNameArgs.push(new StringExp(e.loc, "core.interpolation"));
+            id = new DotTemplateInstanceExp(e.loc, id, Id.imported, moduleNameArgs);
+            auto tiargs = new Objects();
+            auto templateStringArg = new StringExp(e.loc, arg);
+            // banning those instead of forwarding them
+            // templateStringArg.postfix = e.postfix; // forward the postfix to these literals
+            tiargs.push(templateStringArg);
+            id = new DotTemplateInstanceExp(e.loc, id, which, tiargs);
+            id = new CallExp(e.loc, id, new Expressions());
+            return id;
+        }
+
+        auto arguments = new Expressions();
+        arguments.push(makeNonTemplateItem(Id.InterpolationHeader));
+
+        foreach (idx, str; e.interpolatedSet.parts)
+        {
+            if (idx % 2 == 0)
+            {
+                if (str.length > 0)
+                    arguments.push(makeTemplateItem(Id.InterpolatedLiteral, str));
+            }
+            else
+            {
+                arguments.push(makeTemplateItem(Id.InterpolatedExpression, str));
+                Expressions* mix = new Expressions();
+                mix.push(new StringExp(e.loc, str));
+                // FIXME: i'd rather not use MixinExp but idk how to do it lol
+                arguments.push(new MixinExp(e.loc, mix));
+            }
+        }
+
+        arguments.push(makeNonTemplateItem(Id.InterpolationFooter));
+
+        auto loweredTo = new TupleExp(e.loc, arguments);
+        visit(loweredTo);
+
+        result = loweredTo;
+    }
+
     override void visit(StringExp e)
     {
         static if (LOGSEMANTIC)