Improve forward and backward-compatibility for serialized scripts

ARES.bt

@@ -4,11 +4,12 @@
 //      File: ARES.bt
 //   Authors: Harold Cindy
 //   Version: 
-//   Purpose: dissecting serialized Ares chunks
-//  Category: 
-// File Mask: *.ares
-//  ID Bytes: 41 52 45 53
-//   History: 
+//   Purpose: dissecting serialized Ares chunks, bare or inside an
+//            Executor state payload
+//  Category:
+// File Mask: *.ares,*.state
+//  ID Bytes: 41 52 45 53, 45 58 45 43
+//   History:
 //------------------------------------------------
 // This file originally based on Eris' FILEFORMAT file
 
@@ -21,6 +22,7 @@ typedef uint32 uint32_t;
 typedef uint64 uint64_t;
 typedef int16 int16_t;
 typedef int32 int32_t;
+typedef int64 int64_t;
 // Note that in the context of Ares size_t is _always_ 64-bit!
 typedef uint64 size_t;
 typedef size_t ProtoPtr;
@@ -103,6 +105,7 @@ struct PermKey;
 struct Upvaldesc;
 struct LocVar;
 struct CallInfo;
+struct ExecString;
 
 
 // From LuauBytecode.bt
@@ -500,13 +503,26 @@ local uint64 refNum = 0;
 // tracks where we saw each reference
 local uint64 refPositions[0xFFFF] = {0};
 
+/* Closes a length-prefixed record: anything left before record_end is test
+ * padding or fields a newer minor appended, both of which the reader skips.
+ * Declarations leak into the calling struct. */
+void ParseRecordEnd(int64 record_end) {
+    if (FTell() < record_end)
+        uchar appended[record_end - FTell()] <bgcolor=cSilver>;
+    Assert(FTell() == record_end);
+}
+
 typedef struct {
     char sig[4] <bgcolor=0x33aaff>;   /* Header signature for rudimentary validation */
     Assert(sig == "ARES");
-    uint32_t version;
-    /* Version 4 moved the type tags off of lua_Type and onto AresType, so
-     * older blobs cannot be parsed with this template. */
-    Assert(version >= 4);
+    uint32_t major;
+    uint32_t minor;
+    /* Mirrors ARES_FORMAT_MAJOR in VM/include/lua.h. Any minor under it
+     * parses: fields are only ever appended inside records, and each record's
+     * length word steps over what this template doesn't know. */
+    Assert(major == 6);
+    uint32_t record_len;
+    local int64 record_end = FTell() + record_len;
     uint8_t sizeof_number;  /* sizeof(lua_Number) to check type compatibility */
     lua_Number test;    /* -1.234567890 to check representation compatibility */
     uint8_t sizeof_int; /* sizeof(int) in persisted data */
@@ -515,11 +531,12 @@ typedef struct {
     /* Note that the last two fields determine the size of the int and size_t
      * fields in the following definitions. We write each value in the native
      * "size" and check for truncation when reading, if necessary. */
+    /* Reserved when the header is written and patched in after the root
+     * object, see p_header_refcount() in ares.cpp. */
+    uint32_t final_refcount;
+    ParseRecordEnd(record_end);
 } Header <bgcolor=cLtRed>;
 
-// Define this up here so structs can reference the version
-Header header;      /* The header used for basic validation. */
-
 // GC object header with memcat.
 // Declarations leak into the calling struct.
 void ParseGCHeader() {
@@ -554,9 +571,10 @@ typedef struct {
         case ARES_T_VECTOR:
         // storing a reference to a reference? no.
         case ARES_T_REFERENCE:
-        // permanent never writes to the ref table, what would be the point?
-        case ARES_T_PERMANENT:
             break;
+        // Permanents take a number too: persist_keyed() allocates it before
+        // the permanents table gets a look, and u_permanent() reserves it
+        // before reading the key, so the key's own objects number after it.
         default: {
             ourRef = ++refNum;
             // track where we saw this so we can show the address of what's
@@ -565,6 +583,23 @@ typedef struct {
         }
     }
 
+    // Mirrors type_is_framed() in ares.cpp: the VM-shaped bodies carry a
+    // length word so a reader can step over fields it doesn't know.
+    local int64 record_end = -1;
+    switch(type) {
+        case ARES_T_TABLE:
+        case ARES_T_FUNCTION:
+        case ARES_T_USERDATA:
+        case ARES_T_THREAD:
+        case ARES_T_PROTO:
+        case ARES_T_UPVAL:
+        case ARES_T_CLASS:
+        case ARES_T_OBJECT:
+            uint32_t record_len <bgcolor=cLtGray>;
+            record_end = FTell() + record_len;
+            break;
+    }
+
     switch(type) {
         case ARES_T_NIL:
             break;
@@ -611,22 +646,29 @@ typedef struct {
             uint32 reference <bgcolor=cLtGreen>; /* The index the object was referenced with */
             ourRef = reference;
             break;
+        case ARES_T_CLASS:
+        case ARES_T_OBJECT:
+            /* Framed, but ares.cpp has no body for these yet. */
+            Assert(0);
         default:
             Assert(0);
     }
+
+    if (record_end >= 0)
+        ParseRecordEnd(record_end);
 } Object<optimize=false, bgcolor=cLtPurple, read=ReadObject>;
 
 typedef struct {
     ParseGCHeader();
     size_t length;      /* The length of the string */
     char str[length];   /* The actual string (not always null terminated) */
-} String <read=this.str>;
+} String <read=(exists(this.str) ? this.str : "")>;
 
 typedef struct {
     ParseGCHeader();
     size_t length;      /* The length of the buffer */
     char data[length];   /* The actual buffer data */
-} Buffer <read=this.data>;
+} Buffer <read=(exists(this.data) ? this.data : "")>;
 
 struct Table {
     ParseGCHeader();
@@ -782,19 +824,20 @@ struct Proto {
 struct Thread {
     ParseGCHeader();
     Object env;
-    int stacksize;      /* The overall size of the stack filled with objects,
-                         * including all stack frames. */
+    uint32_t stacksize; /* Allocated stack slots, restored as-is */
     size_t top;         /* top = L->top - L->stack; */
     Object stack[top];    /* All stack values, bottom up */
 
     AresStatus status;     /* current thread status (ok, yield) */
     uint8_t activememcat <bgcolor=cLtYellow>;
-    size_t errfunc;     /* NOT USED current error handling function (stack index) */
     Object namecall;    /* The pending namecall string, nil for none */
 
-    int32_t num_cis;        /* number of callinfo frames */
-    /* The CallInfo stack, starting with base_ci */
+    uint32_t size_ci;   /* Allocated callinfo slots, never below BASIC_CI_SIZE */
+    uint32_t num_cis;   /* number of callinfo frames */
+    /* The CallInfo stack, starting with base_ci. Each frame is its own record. */
     struct CallInfo {
+        uint32_t record_len <bgcolor=cLtGray>;
+        local int64 record_end = FTell() + record_len;
         size_t func;        /* func = ci->func - thread->stack */
         size_t top;         /* top = ci->top - thread->stack */
         size_t base;        /* base = ci->base - thread-stack */
@@ -806,22 +849,14 @@ struct Thread {
             int yield_point;
             int savedpc; /* savedpc = ci->u.l.savedpc - ci_func(ci)->p->code */
         } else if (ci_kind == ERIS_CI_KIND_C) {
-            //uint8_t status;
-            //if (callstatus & (CIST_YPCALL | CIST_YIELDED)) {
-            //    int32_t ctx;  /* context info. in case of yields */
-            //    Object k;     /* C function, callback for resuming */
-            //}
+            int32_t errfunc;    /* pcall's error handler, 1-based from ci->base, 0 for none */
             Object function;
         } else {
             Assert(ci_kind == ERIS_CI_KIND_NONE);
         }
+        ParseRecordEnd(record_end);
     } ci[num_cis] <bgcolor=cLtAqua, optimize=false>;
 
-    if (status == ARES_S_YIELD) {
-        // size_t extra;   /* value of thread->ci->extra, which is the original
-        //                 * value of thread->ci->func */
-    }
-
     while (TRUE) {
         struct OpenUpval {
             size_t idx;         /* stack index of the value + 1; 0 if end of list */
@@ -841,4 +876,83 @@ struct PermKey {
      * value in the permanents table when unpersisting has the correct type. */
 };
 
-Object rootobj;     /* The root object that was persisted. */
+typedef struct {
+    refNum = 0;
+    Header header;
+    Object rootobj;     /* The root object that was persisted. */
+    /* The same check the reader makes in u_finish() */
+    if (refNum != header.final_refcount)
+    {
+        Printf("refNum != header.final_refcount, %d != %d\n", refNum, header.final_refcount);
+        Assert(0);
+    }
+} AresStream;
+
+/* Length-prefixed string as written by ByteWriter::writeString */
+typedef struct {
+    uint32_t length;
+    char str[length];
+} ExecString <read=(exists(this.str) ? this.str : "")>;
+
+/* The wrapper Script::serializeState() puts around an Ares blob, see
+ * Executor/src/Script.cpp. Fields only ever get appended inside the two
+ * sections, so a newer minor parses. */
+typedef struct {
+    /* The base class's fingerprint, covering the core section. Mirrors
+     * kScriptStateFingerprint in Executor/include/Luau/Script.h. */
+    char tag[4] <bgcolor=0x33aaff>;
+    Assert(tag == "EXEC");
+    uint32_t major;
+    uint32_t minor;
+    Assert(major == 2);
+    /* The concrete Script subclass's fingerprint (EXEC again for the base
+     * class), covering the extra section at the end. */
+    char class_tag[4] <bgcolor=0x33aaff>;
+    uint32_t class_major;
+    uint32_t class_minor;
+
+    struct CoreSection {
+        uint32_t record_len <bgcolor=cLtGray>;
+        local int64 record_end = FTell() + record_len;
+        float sleep;
+        uint32_t memory_limit;
+        uint8_t fault_kind;
+        ExecString fault_string;
+        ExecString extended_fault_string;
+        uint8_t main_function_complete;
+        uint8_t is_lsl;
+        uint32_t api_version;
+        /* The indra registers, opaque bitfields in the asset header's mask
+         * space (bit n-1 is builtins.txt event n). */
+        int32_t current_state;
+        int32_t next_state;
+        uint64_t current_handler;
+        uint64_t sticky_handler;
+        uint64_t current_events;
+        uint64_t event_handlers;
+        ParseRecordEnd(record_end);
+    } core <bgcolor=cLtGreen>;
+
+    uint32_t ares_length;
+    local int64 ares_end = FTell() + ares_length;
+    AresStream ares;
+    Assert(FTell() == ares_end);
+
+    /* Subclass state, versioned by class_major/class_minor. The base Script
+     * writes nothing here, so the layout is whatever serializeExtra() chose. */
+    struct ExtraSection {
+        uint32_t record_len <bgcolor=cLtGray>;
+        if (record_len > 0)
+            uchar data[record_len];
+    } extra;
+} ExecState;
+
+local string magic = ReadString(0, 4);
+if (magic == "EXEC")
+    ExecState exec;
+else if (magic == "ARES")
+    AresStream ares;
+else
+    Assert(0);
+/* Both readers refuse trailing bytes */
+Assert(FEof());

Bytecode/include/Luau/BytecodeHeader.h

@@ -0,0 +1,42 @@
+// ServerLua: the asset format, a BytecodeHeader followed by Luau bytecode.
+#pragma once
+
+#include "Luau/ByteStream.h"
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+namespace Luau
+{
+
+// What a host stores in front of compiled bytecode in a script asset. Carries
+// what has to be known about the script before an image exists, so nothing is
+// rediscovered from the VM. The codec is declared just below.
+struct BytecodeHeader
+{
+    bool isLSL = false;
+    uint32_t apiVersion = 0;
+    // Per-state LSL handler masks, indexed by state number, bit (event index - 1)
+    // with the index being the event's position in builtins.txt (see
+    // LSLBuiltins.h). Empty for SLua.
+    std::vector<uint64_t> stateHandlerMasks;
+    // Bytes charged per script for the bytecode, so we can swap bytecode behind
+    // people's backs without moving reported memory. Zero charges the real length.
+    uint32_t chargedBytecodeSize = 0;
+};
+
+// Numbered above the header versions the server wrote privately before this
+// codec existed, so the two never read as each other in a log line. Fields
+// only ever get appended inside the section, so any minor under the major
+// parses and the reader defaults what an older writer left out.
+constexpr StateFingerprint kBytecodeHeaderFingerprint{{'L', 'U', 'A', 'U'}, 6, 0};
+
+// Appends the header to `out`; the raw bytecode follows it
+void writeBytecodeHeader(std::string& out, const BytecodeHeader& header);
+
+// Parses the header off the front of an asset. `bytecode_start` is where the
+// raw bytecode begins. False for a wrong tag, a different major, or truncation.
+bool readBytecodeHeader(const char* data, size_t len, BytecodeHeader& header, size_t& bytecode_start);
+
+} // namespace Luau

Bytecode/src/BytecodeHeader.cpp

@@ -0,0 +1,70 @@
+// ServerLua: asset header codec, see `BytecodeHeader` in BytecodeHeader.h
+#include "Luau/BytecodeHeader.h"
+
+#include <cstring>
+
+namespace Luau
+{
+
+void writeBytecodeHeader(std::string& out, const BytecodeHeader& header)
+{
+    ByteWriter writer{out};
+    writer.writeBytes(kBytecodeHeaderFingerprint.tag, sizeof(kBytecodeHeaderFingerprint.tag));
+    writer.writeU32(kBytecodeHeaderFingerprint.major);
+    writer.writeU32(kBytecodeHeaderFingerprint.minor);
+
+    size_t section = writer.beginSection();
+    writer.writeU8((uint8_t)header.isLSL);
+    writer.writeU32(header.apiVersion);
+    writer.writeU32((uint32_t)header.stateHandlerMasks.size());
+    for (uint64_t mask : header.stateHandlerMasks)
+        writer.writeU64(mask);
+    writer.writeU32(header.chargedBytecodeSize);
+    // New fields go here, and bump kBytecodeHeaderFingerprint.minor
+    writer.endSection(section);
+}
+
+bool readBytecodeHeader(const char* data, size_t len, BytecodeHeader& header, size_t& bytecode_start)
+{
+    if (data == nullptr)
+        return false;
+
+    ByteReader reader{data, len};
+
+    char tag[sizeof(kBytecodeHeaderFingerprint.tag)];
+    if (!reader.readBytes(tag, sizeof(tag)) || memcmp(tag, kBytecodeHeaderFingerprint.tag, sizeof(tag)) != 0)
+        return false;
+
+    uint32_t major = 0;
+    uint32_t minor = 0;
+    if (!reader.readU32(major) || !reader.readU32(minor) || major != kBytecodeHeaderFingerprint.major)
+        return false;
+
+    ByteReader section{nullptr, 0};
+    if (!reader.readSection(section))
+        return false;
+
+    uint8_t is_lsl = 0;
+    uint32_t num_states = 0;
+    if (!section.readU8(is_lsl) || !section.readU32(header.apiVersion) || !section.readU32(num_states))
+        return false;
+    // Eight bytes each, so a count the section can't hold is a corrupt header
+    // rather than something to allocate for
+    if (num_states > section.remaining / sizeof(uint64_t))
+        return false;
+    header.isLSL = is_lsl != 0;
+    header.stateHandlerMasks.resize(num_states);
+    for (uint64_t& mask : header.stateHandlerMasks)
+    {
+        if (!section.readU64(mask))
+            return false;
+    }
+    if (!section.readU32(header.chargedBytecodeSize))
+        return false;
+    // Fields appended after 6.0 are read here only if the section has them
+
+    bytecode_start = len - reader.remaining;
+    return true;
+}
+
+} // namespace Luau