From 7fc92bc2366b28997d37dc02c612eddcf5f18057 Mon Sep 17 00:00:00 2001 From: Hani Nemati Date: Tue, 29 Sep 2026 17:16:47 -0700 Subject: [PATCH 01/14] Support CTF 2 LTTng traces LTTng 2.15+ records CTF 2 (JSON metadata) by default, which the TSDL parser could not read. Add a CTF 2 metadata parser that maps CTF 2 field classes onto the existing descriptor model, and select the parser from the metadata content. Also fix issues that CTF 2 / multi-channel LTTng traces exposed: - Accept any _ stream file, not only chan* - Event ids are unique per stream: look up event descriptors and generic event kinds by (stream, id) - Look up streams by id and read the packet context after determining the stream id - Decode explicitly big-endian whole-byte integers - Read the event-specific context when one is defined Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CtfPlayback/EventStreams/CtfEvent.cs | 9 +- CtfPlayback/EventStreams/CtfPacket.cs | 8 +- .../EventStreams/Interfaces/ICtfEvent.cs | 5 + CtfPlayback/Metadata/Ctf2/Ctf2Json.cs | 273 ++++++ .../Metadata/Ctf2/Ctf2MetadataParser.cs | 792 ++++++++++++++++++ .../Ctf2VariableLengthIntegerDescriptor.cs | 86 ++ CtfPlayback/Metadata/CtfMetadataExtensions.cs | 33 + CtfPlayback/Metadata/CtfMetadataText.cs | 62 ++ .../CtfVersionDetectingMetadataParser.cs | 62 ++ .../Metadata/Types/CtfIntegerDescriptor.cs | 14 + CtfUnitTest/Ctf2MetadataTests.cs | 281 +++++++ LTTngCds/CookerData/LTTngEvent.cs | 7 +- .../Descriptors/EventDescriptor.cs | 9 +- .../FolderInput/LTTngFolderInput.cs | 2 +- LTTngCds/CtfExtensions/LTTngMetadata.cs | 12 +- .../LTTngPlaybackCustomization.cs | 14 +- LTTngCds/CtfExtensions/LTTngStreamFiles.cs | 41 + .../ZipArchiveInput/LTTngZipArchiveInput.cs | 2 +- LTTngDataExtUnitTest/LTTngCtf2UnitTest.cs | 221 +++++ .../DataOutputTypes/LTTngGenericEvent.cs | 30 +- LinuxTraceLogCapture.md | 10 +- 21 files changed, 1946 insertions(+), 27 deletions(-) create mode 100644 CtfPlayback/Metadata/Ctf2/Ctf2Json.cs create mode 100644 CtfPlayback/Metadata/Ctf2/Ctf2MetadataParser.cs create mode 100644 CtfPlayback/Metadata/Ctf2/Ctf2VariableLengthIntegerDescriptor.cs create mode 100644 CtfPlayback/Metadata/CtfMetadataExtensions.cs create mode 100644 CtfPlayback/Metadata/CtfMetadataText.cs create mode 100644 CtfPlayback/Metadata/CtfVersionDetectingMetadataParser.cs create mode 100644 CtfUnitTest/Ctf2MetadataTests.cs create mode 100644 LTTngCds/CtfExtensions/LTTngStreamFiles.cs create mode 100644 LTTngDataExtUnitTest/LTTngCtf2UnitTest.cs diff --git a/CtfPlayback/EventStreams/CtfEvent.cs b/CtfPlayback/EventStreams/CtfEvent.cs index 73aa195..0be1fbb 100644 --- a/CtfPlayback/EventStreams/CtfEvent.cs +++ b/CtfPlayback/EventStreams/CtfEvent.cs @@ -5,6 +5,7 @@ using System.Diagnostics; using CtfPlayback.EventStreams.Interfaces; using CtfPlayback.FieldValues; +using CtfPlayback.Metadata; using CtfPlayback.Metadata.Interfaces; namespace CtfPlayback.EventStreams @@ -25,6 +26,8 @@ public CtfEvent(PacketReader packetReader, ICtfMetadata metadata, ICtfPacket own public ulong ByteOffsetWithinPacket { get; private set; } + public uint StreamId => this.owningPacket.StreamId; + public CtfTimestamp Timestamp { get; private set; } public CtfFieldValue StreamDefinedEventHeader { get; private set; } @@ -102,8 +105,7 @@ public void Read() private void ReadStreamEventHeader() { - uint streamIndex = this.owningPacket.StreamId; - var eventHeaderDescriptor = this.packetReader.Metadata.Streams[(int)streamIndex].EventHeader; + var eventHeaderDescriptor = this.packetReader.Metadata.GetStream(this.owningPacket.StreamId).EventHeader; Debug.Assert(eventHeaderDescriptor != null); StreamDefinedEventHeader = eventHeaderDescriptor.Read(this.packetReader); @@ -111,8 +113,7 @@ private void ReadStreamEventHeader() private void ReadStreamEventContext() { - uint streamIndex = this.owningPacket.StreamId; - var eventContextDescriptor = this.packetReader.Metadata.Streams[(int)streamIndex].EventContext; + var eventContextDescriptor = this.packetReader.Metadata.GetStream(this.owningPacket.StreamId).EventContext; if (eventContextDescriptor == null) { return; diff --git a/CtfPlayback/EventStreams/CtfPacket.cs b/CtfPlayback/EventStreams/CtfPacket.cs index a7c9d00..39a59f5 100644 --- a/CtfPlayback/EventStreams/CtfPacket.cs +++ b/CtfPlayback/EventStreams/CtfPacket.cs @@ -6,6 +6,7 @@ using System.IO; using CtfPlayback.EventStreams.Interfaces; using CtfPlayback.FieldValues; +using CtfPlayback.Metadata; using CtfPlayback.Metadata.Interfaces; namespace CtfPlayback.EventStreams @@ -61,10 +62,12 @@ public bool ReadPacketMetadata() this.PacketByteOffset = this.packetReader.CountOfBytesProcessed; ReadPacketHeader(); - ReadPacketContext(); + // The packet context layout depends on the stream, which is identified in the packet header. this.DetermineStreamIndex(); + ReadPacketContext(); + this.packetReader.SetPacketSize(this.packetReader.PlaybackCustomization.GetBitsInPacket(this)); this.BitsInPacketContent = @@ -116,7 +119,7 @@ private void ReadPacketHeader() private void ReadPacketContext() { - this.StreamPacketContext = this.Metadata.Streams[(int)this.StreamId].PacketContext.Read(this.packetReader) as CtfStructValue; + this.StreamPacketContext = this.Metadata.GetStream(this.StreamId).PacketContext.Read(this.packetReader) as CtfStructValue; } private ulong GetRemainingBits() @@ -135,6 +138,7 @@ private void DetermineStreamIndex() if (streamIdDescriptor == null) { this.StreamId = 0; + return; } bool streamIdFound = this.TracePacketHeader.FieldsByName.TryGetValue("stream_id", out var streamIdFieldValue); diff --git a/CtfPlayback/EventStreams/Interfaces/ICtfEvent.cs b/CtfPlayback/EventStreams/Interfaces/ICtfEvent.cs index c80a429..3775772 100644 --- a/CtfPlayback/EventStreams/Interfaces/ICtfEvent.cs +++ b/CtfPlayback/EventStreams/Interfaces/ICtfEvent.cs @@ -16,6 +16,11 @@ public interface ICtfEvent /// ulong ByteOffsetWithinPacket { get; } + /// + /// Id of the stream (stream class) that contains this event. + /// + uint StreamId { get; } + /// /// Timestamp associated with the event /// diff --git a/CtfPlayback/Metadata/Ctf2/Ctf2Json.cs b/CtfPlayback/Metadata/Ctf2/Ctf2Json.cs new file mode 100644 index 0000000..9e0cf89 --- /dev/null +++ b/CtfPlayback/Metadata/Ctf2/Ctf2Json.cs @@ -0,0 +1,273 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text; + +namespace CtfPlayback.Metadata.Ctf2 +{ + /// + /// A JSON number, kept in its textual form so that 64-bit unsigned values are not truncated. + /// + internal sealed class Ctf2JsonNumber + { + internal Ctf2JsonNumber(string text) + { + this.Text = text; + } + + internal string Text { get; } + + internal bool IsNegative => this.Text.StartsWith("-", StringComparison.Ordinal); + + internal long AsLong() => long.Parse(this.Text, NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture); + + internal ulong AsULong() => ulong.Parse(this.Text, NumberStyles.None, CultureInfo.InvariantCulture); + + public override string ToString() => this.Text; + } + + /// + /// Minimal JSON reader used for CTF 2 metadata. Objects are returned as + /// of string to object, arrays as of object, + /// numbers as , and strings, booleans and null as their .NET equivalents. + /// + internal sealed class Ctf2Json + { + private readonly string text; + private int position; + + private Ctf2Json(string text) + { + this.text = text; + } + + internal static object Parse(string text) + { + var reader = new Ctf2Json(text); + reader.SkipWhitespace(); + var value = reader.ReadValue(); + reader.SkipWhitespace(); + if (reader.position != text.Length) + { + throw reader.Error("Unexpected trailing characters"); + } + + return value; + } + + private object ReadValue() + { + if (this.position >= this.text.Length) + { + throw this.Error("Unexpected end of JSON"); + } + + char c = this.text[this.position]; + switch (c) + { + case '{': + return this.ReadObject(); + case '[': + return this.ReadArray(); + case '"': + return this.ReadString(); + case 't': + this.Expect("true"); + return true; + case 'f': + this.Expect("false"); + return false; + case 'n': + this.Expect("null"); + return null; + default: + if (c == '-' || (c >= '0' && c <= '9')) + { + return this.ReadNumber(); + } + + throw this.Error($"Unexpected character '{c}'"); + } + } + + private Dictionary ReadObject() + { + var result = new Dictionary(StringComparer.Ordinal); + this.position++; + this.SkipWhitespace(); + if (this.TryConsume('}')) + { + return result; + } + + while (true) + { + this.SkipWhitespace(); + if (this.position >= this.text.Length || this.text[this.position] != '"') + { + throw this.Error("Expected property name"); + } + + string name = this.ReadString(); + this.SkipWhitespace(); + if (!this.TryConsume(':')) + { + throw this.Error("Expected ':'"); + } + + this.SkipWhitespace(); + result[name] = this.ReadValue(); + this.SkipWhitespace(); + + if (this.TryConsume(',')) + { + continue; + } + + if (this.TryConsume('}')) + { + return result; + } + + throw this.Error("Expected ',' or '}'"); + } + } + + private List ReadArray() + { + var result = new List(); + this.position++; + this.SkipWhitespace(); + if (this.TryConsume(']')) + { + return result; + } + + while (true) + { + this.SkipWhitespace(); + result.Add(this.ReadValue()); + this.SkipWhitespace(); + + if (this.TryConsume(',')) + { + continue; + } + + if (this.TryConsume(']')) + { + return result; + } + + throw this.Error("Expected ',' or ']'"); + } + } + + private string ReadString() + { + this.position++; + var sb = new StringBuilder(); + while (this.position < this.text.Length) + { + char c = this.text[this.position++]; + if (c == '"') + { + return sb.ToString(); + } + + if (c != '\\') + { + sb.Append(c); + continue; + } + + if (this.position >= this.text.Length) + { + break; + } + + char escaped = this.text[this.position++]; + switch (escaped) + { + case '"': sb.Append('"'); break; + case '\\': sb.Append('\\'); break; + case '/': sb.Append('/'); break; + case 'b': sb.Append('\b'); break; + case 'f': sb.Append('\f'); break; + case 'n': sb.Append('\n'); break; + case 'r': sb.Append('\r'); break; + case 't': sb.Append('\t'); break; + case 'u': + if (this.position + 4 > this.text.Length || + !int.TryParse(this.text.Substring(this.position, 4), NumberStyles.HexNumber, CultureInfo.InvariantCulture, out int codeUnit)) + { + throw this.Error("Invalid unicode escape"); + } + + sb.Append((char)codeUnit); + this.position += 4; + break; + default: + throw this.Error($"Invalid escape character '{escaped}'"); + } + } + + throw this.Error("Unterminated string"); + } + + private Ctf2JsonNumber ReadNumber() + { + int start = this.position; + while (this.position < this.text.Length) + { + char c = this.text[this.position]; + if ((c >= '0' && c <= '9') || c == '-' || c == '+' || c == '.' || c == 'e' || c == 'E') + { + this.position++; + continue; + } + + break; + } + + return new Ctf2JsonNumber(this.text.Substring(start, this.position - start)); + } + + private void Expect(string literal) + { + if (string.CompareOrdinal(this.text, this.position, literal, 0, literal.Length) != 0) + { + throw this.Error($"Expected '{literal}'"); + } + + this.position += literal.Length; + } + + private bool TryConsume(char c) + { + if (this.position < this.text.Length && this.text[this.position] == c) + { + this.position++; + return true; + } + + return false; + } + + private void SkipWhitespace() + { + while (this.position < this.text.Length && char.IsWhiteSpace(this.text[this.position])) + { + this.position++; + } + } + + private CtfMetadataException Error(string message) + { + return new CtfMetadataException($"Invalid CTF 2 metadata JSON: {message} at offset {this.position}."); + } + } +} diff --git a/CtfPlayback/Metadata/Ctf2/Ctf2MetadataParser.cs b/CtfPlayback/Metadata/Ctf2/Ctf2MetadataParser.cs new file mode 100644 index 0000000..306606e --- /dev/null +++ b/CtfPlayback/Metadata/Ctf2/Ctf2MetadataParser.cs @@ -0,0 +1,792 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using CtfPlayback.Metadata.Helpers; +using CtfPlayback.Metadata.Interfaces; +using CtfPlayback.Metadata.InternalHelpers; +using CtfPlayback.Metadata.NamedScopes; +using CtfPlayback.Metadata.TypeInterfaces; +using CtfPlayback.Metadata.Types; + +namespace CtfPlayback.Metadata.Ctf2 +{ + /// + /// Parses CTF 2 metadata (a JSON text sequence of fragments, see https://diamon.org/ctf/) into the same + /// descriptor model used for CTF 1.8, so that the rest of the playback pipeline is format-agnostic. + /// + public sealed class Ctf2MetadataParser + : ICtfMetadataParser + { + /// + /// Each CTF 2 metadata fragment is preceded by this record separator (RFC 7464). + /// + public const char RecordSeparator = '\u001e'; + + private const string DefaultClockTimestampRole = "default-clock-timestamp"; + private const string PacketEndDefaultClockTimestampRole = "packet-end-default-clock-timestamp"; + + private readonly ICtfMetadataBuilder metadataBuilder; + private readonly bool prefixEventFieldNamesWithUnderscore; + + private readonly Dictionary aliases = new Dictionary(StringComparer.Ordinal); + + /// + /// Constructor. + /// + /// The object used to build metadata + /// + /// CTF 1.8 LTTng traces prefix event context and payload field names with '_'. Set this to produce + /// the same field names from CTF 2 metadata. + /// + public Ctf2MetadataParser(ICtfMetadataBuilder metadataBuilder, bool prefixEventFieldNamesWithUnderscore) + { + this.metadataBuilder = metadataBuilder ?? throw new ArgumentNullException(nameof(metadataBuilder)); + this.prefixEventFieldNamesWithUnderscore = prefixEventFieldNamesWithUnderscore; + } + + /// + /// Determines whether the metadata text is CTF 2 metadata. + /// + /// Metadata text, with any packetization removed + /// true if the text is a CTF 2 JSON text sequence + public static bool IsCtf2Metadata(string metadataText) + { + if (metadataText == null) + { + return false; + } + + foreach (char c in metadataText) + { + if (c == RecordSeparator) + { + return true; + } + + if (!char.IsWhiteSpace(c)) + { + return false; + } + } + + return false; + } + + /// + public ICtfMetadata Parse(Stream metadataStream) + { + if (metadataStream == null) + { + throw new ArgumentNullException(nameof(metadataStream)); + } + + string text = CtfMetadataText.Read(CtfMetadataText.ReadAllBytes(metadataStream)); + return this.Parse(text); + } + + /// + /// Parses CTF 2 metadata text. + /// + /// Metadata text, with any packetization removed + /// The parsed metadata + public ICtfMetadata Parse(string metadataText) + { + var fragments = metadataText + .Split(RecordSeparator) + .Where(fragment => !string.IsNullOrWhiteSpace(fragment)) + .Select(fragment => Ctf2Json.Parse(fragment) as Dictionary + ?? throw new CtfMetadataException("CTF 2 metadata fragment is not a JSON object.")) + .ToList(); + + if (fragments.Count == 0 || GetString(fragments[0], "type") != "preamble") + { + throw new CtfMetadataException("CTF 2 metadata does not start with a preamble fragment."); + } + + var preamble = fragments[0]; + if (!TryGetNumber(preamble, "version", out var version) || version.AsLong() != 2) + { + throw new CtfMetadataException($"Unsupported CTF metadata version: {GetValue(preamble, "version")}."); + } + + Dictionary traceClass = null; + var clockClasses = new List>(); + var dataStreamClasses = new List>(); + var eventRecordClasses = new List>(); + + foreach (var fragment in fragments.Skip(1)) + { + string type = GetString(fragment, "type"); + switch (type) + { + case "field-class-alias": + this.aliases[GetRequiredString(fragment, "name")] = GetRequiredValue(fragment, "field-class"); + break; + case "trace-class": + traceClass = fragment; + break; + case "clock-class": + clockClasses.Add(fragment); + break; + case "data-stream-class": + dataStreamClasses.Add(fragment); + break; + case "event-record-class": + eventRecordClasses.Add(fragment); + break; + case "preamble": + throw new CtfMetadataException("CTF 2 metadata contains more than one preamble fragment."); + default: + // Unknown fragment types are ignored so newer metadata can still be read. + break; + } + } + + if (traceClass == null) + { + throw new CtfMetadataException("CTF 2 metadata does not contain a trace-class fragment."); + } + + this.AddTraceClass(traceClass, preamble); + + foreach (var clockClass in clockClasses) + { + this.metadataBuilder.AddClock(CreateClock(clockClass)); + } + + // Streams are looked up by id, and CTF 1.8 streams are declared in id order. Keep that ordering. + foreach (var dataStreamClass in dataStreamClasses.OrderBy(GetDataStreamClassId)) + { + this.AddDataStreamClass(dataStreamClass); + } + + foreach (var eventRecordClass in eventRecordClasses) + { + this.AddEventRecordClass(eventRecordClass); + } + + return this.metadataBuilder; + } + + private void AddTraceClass(Dictionary traceClass, Dictionary preamble) + { + var packetHeaderClass = GetValue(traceClass, "packet-header-field-class"); + if (packetHeaderClass == null) + { + throw new CtfMetadataException("CTF 2 trace-class does not define a packet header field class."); + } + + if (!(this.BuildFieldClass(packetHeaderClass, BuildContext.Header(null)) is CtfStructDescriptor packetHeader)) + { + throw new CtfMetadataException("CTF 2 trace-class packet header field class is not a structure."); + } + + var traceBag = new CtfPropertyBag(); + traceBag.AddValue("major", "2"); + traceBag.AddValue("minor", "0"); + traceBag.AddValue("byte_order", "le"); + string uid = GetString(traceClass, "uid") ?? FormatUuid(GetValue(preamble, "uuid")); + if (uid != null && Guid.TryParse(uid, out _)) + { + traceBag.AddValue("uuid", uid); + } + + this.metadataBuilder.SetTraceDescriptor(new CtfTraceDescriptor(traceBag, packetHeader)); + + var environmentBag = new CtfPropertyBag(); + if (GetValue(traceClass, "environment") is Dictionary environment) + { + foreach (var entry in environment) + { + environmentBag.AddValue(entry.Key, Convert.ToString(entry.Value, CultureInfo.InvariantCulture)); + } + } + + this.metadataBuilder.SetEnvironmentDescriptor(new CtfEnvironmentDescriptor(environmentBag)); + } + + private static CtfClockDescriptor CreateClock(Dictionary clockClass) + { + // CTF 2 data stream classes refer to clock classes by id; older drafts only had a name. + string clockName = GetString(clockClass, "id") ?? GetRequiredString(clockClass, "name"); + + ulong frequency = TryGetNumber(clockClass, "frequency", out var frequencyNumber) ? frequencyNumber.AsULong() : 1000000000ul; + + var bag = new CtfPropertyBag(); + bag.AddValue("name", clockName); + bag.AddValue("freq", frequency.ToString(CultureInfo.InvariantCulture)); + + string description = GetString(clockClass, "description"); + if (description != null) + { + bag.AddValue("description", description); + } + + string uid = GetString(clockClass, "uid") ?? FormatUuid(GetValue(clockClass, "uuid")); + if (uid != null && Guid.TryParse(uid, out _)) + { + bag.AddValue("uuid", uid); + } + + if (TryGetNumber(clockClass, "precision", out var precision) && !precision.IsNegative) + { + bag.AddValue("precision", precision.Text); + } + + if (GetValue(clockClass, "offset-from-origin") is Dictionary offset) + { + // The descriptor expects the full offset in clock cycles. + decimal cycles = 0; + if (TryGetNumber(offset, "seconds", out var seconds)) + { + cycles += decimal.Parse(seconds.Text, CultureInfo.InvariantCulture) * frequency; + } + + if (TryGetNumber(offset, "cycles", out var offsetCycles)) + { + cycles += decimal.Parse(offsetCycles.Text, CultureInfo.InvariantCulture); + } + + if (cycles > 0) + { + bag.AddValue("offset", decimal.ToUInt64(cycles).ToString(CultureInfo.InvariantCulture)); + } + } + + bag.AddValue("absolute", GetValue(clockClass, "origin") != null ? "true" : "false"); + + return new CtfClockDescriptor(bag); + } + + private void AddDataStreamClass(Dictionary dataStreamClass) + { + uint id = GetDataStreamClassId(dataStreamClass); + string clockName = GetString(dataStreamClass, "default-clock-class-id"); + + var packetContextClass = GetValue(dataStreamClass, "packet-context-field-class") + ?? throw new CtfMetadataException($"CTF 2 data stream class {id} does not define a packet context."); + var eventHeaderClass = GetValue(dataStreamClass, "event-record-header-field-class") + ?? throw new CtfMetadataException($"CTF 2 data stream class {id} does not define an event record header."); + + var headerContext = BuildContext.Header(clockName); + + if (!(this.BuildFieldClass(packetContextClass, headerContext) is CtfStructDescriptor packetContext)) + { + throw new CtfMetadataException($"CTF 2 data stream class {id} packet context is not a structure."); + } + + if (!(this.BuildFieldClass(eventHeaderClass, headerContext) is CtfStructDescriptor eventHeader)) + { + throw new CtfMetadataException($"CTF 2 data stream class {id} event record header is not a structure."); + } + + CtfStructDescriptor eventContext = null; + var eventContextClass = GetValue(dataStreamClass, "event-record-common-context-field-class"); + if (eventContextClass != null) + { + eventContext = this.BuildFieldClass(eventContextClass, this.EventContext(clockName)) as CtfStructDescriptor + ?? throw new CtfMetadataException($"CTF 2 data stream class {id} event record common context is not a structure."); + } + + var bag = new CtfPropertyBag(); + bag.AddValue("id", id.ToString(CultureInfo.InvariantCulture)); + + this.metadataBuilder.AddStream(new CtfStreamDescriptor(bag, eventHeader, eventContext, packetContext)); + } + + private void AddEventRecordClass(Dictionary eventRecordClass) + { + string name = GetString(eventRecordClass, "name") ?? string.Empty; + try + { + uint streamId = TryGetNumber(eventRecordClass, "data-stream-class-id", out var streamIdNumber) ? (uint)streamIdNumber.AsULong() : 0; + ulong id = TryGetNumber(eventRecordClass, "id", out var idNumber) ? idNumber.AsULong() : 0; + + var assignments = new Dictionary(StringComparer.Ordinal) + { + ["name"] = name, + ["id"] = id.ToString(CultureInfo.InvariantCulture), + ["stream_id"] = streamId.ToString(CultureInfo.InvariantCulture), + }; + + if (TryGetLogLevel(eventRecordClass, out string logLevel)) + { + assignments["loglevel"] = logLevel; + } + + var context = this.EventContext(null); + var typeDeclarations = new Dictionary(StringComparer.Ordinal); + + var payloadClass = GetValue(eventRecordClass, "payload-field-class"); + typeDeclarations["fields"] = payloadClass != null + ? this.BuildFieldClass(payloadClass, context) + : new CtfStructDescriptor(new CtfPropertyBag(), Array.Empty()); + + var specificContextClass = GetValue(eventRecordClass, "specific-context-field-class"); + if (specificContextClass != null) + { + typeDeclarations["context"] = this.BuildFieldClass(specificContextClass, context); + } + + this.metadataBuilder.AddEvent(assignments, typeDeclarations); + } + catch (Exception e) when (!(e is CtfMetadataException)) + { + throw new CtfMetadataException($"Unable to process CTF 2 event record class '{name}': {e.Message}", e); + } + } + + private BuildContext EventContext(string clockName) + { + return new BuildContext(clockName, this.prefixEventFieldNamesWithUnderscore); + } + + private CtfMetadataTypeDescriptor BuildFieldClass(object fieldClass, BuildContext context, IReadOnlyList selectorOptions = null) + { + var fieldClassObject = this.ResolveFieldClass(fieldClass); + string type = GetRequiredString(fieldClassObject, "type"); + + switch (type) + { + case "fixed-length-unsigned-integer": + case "fixed-length-bit-array": + case "fixed-length-bit-map": + case "fixed-length-boolean": + return BuildFixedLengthInteger(fieldClassObject, false, context, selectorOptions); + + case "fixed-length-signed-integer": + return BuildFixedLengthInteger(fieldClassObject, true, context, selectorOptions); + + case "variable-length-unsigned-integer": + case "variable-length-signed-integer": + if (selectorOptions != null) + { + throw new CtfMetadataException("Variable-length integer variant selectors are not supported."); + } + + return new Ctf2VariableLengthIntegerDescriptor( + type == "variable-length-signed-integer", + GetDisplayBase(fieldClassObject), + GetClockMap(fieldClassObject, context)); + + case "fixed-length-floating-point-number": + case "fixed-length-floating-point": + return BuildFloatingPoint(fieldClassObject); + + case "null-terminated-string": + EnsureUtf8(fieldClassObject); + var stringBag = new CtfPropertyBag(); + stringBag.AddValue("encoding", "UTF8"); + return new CtfStringDescriptor(stringBag); + + case "static-length-string": + EnsureUtf8(fieldClassObject); + return new CtfArrayDescriptor(CreateByte("UTF8", 10), GetRequiredNumber(fieldClassObject, "length").AsULong().ToString(CultureInfo.InvariantCulture)); + + case "dynamic-length-string": + EnsureUtf8(fieldClassObject); + return new CtfArrayDescriptor(CreateByte("UTF8", 10), this.GetFieldLocation(fieldClassObject, "length-field-location", context)); + + case "static-length-blob": + return new CtfArrayDescriptor(CreateByte(null, 16), GetRequiredNumber(fieldClassObject, "length").AsULong().ToString(CultureInfo.InvariantCulture)); + + case "dynamic-length-blob": + return new CtfArrayDescriptor(CreateByte(null, 16), this.GetFieldLocation(fieldClassObject, "length-field-location", context)); + + case "static-length-array": + return new CtfArrayDescriptor( + this.BuildArrayElement(fieldClassObject, context), + GetRequiredNumber(fieldClassObject, "length").AsULong().ToString(CultureInfo.InvariantCulture)); + + case "dynamic-length-array": + return new CtfArrayDescriptor( + this.BuildArrayElement(fieldClassObject, context), + this.GetFieldLocation(fieldClassObject, "length-field-location", context)); + + case "structure": + return this.BuildStructure(fieldClassObject, context); + + case "variant": + return this.BuildVariant(fieldClassObject, context); + + default: + throw new CtfMetadataException($"CTF 2 field class type '{type}' is not supported."); + } + } + + private Dictionary ResolveFieldClass(object fieldClass) + { + var seen = new HashSet(StringComparer.Ordinal); + while (fieldClass is string aliasName) + { + if (!seen.Add(aliasName) || !this.aliases.TryGetValue(aliasName, out fieldClass)) + { + throw new CtfMetadataException($"Unknown CTF 2 field class alias '{aliasName}'."); + } + } + + return fieldClass as Dictionary + ?? throw new CtfMetadataException("CTF 2 field class is not a JSON object."); + } + + private static CtfMetadataTypeDescriptor BuildFixedLengthInteger( + Dictionary fieldClass, + bool signed, + BuildContext context, + IReadOnlyList selectorOptions) + { + // Like the CTF 1.8 path, only whole-byte big-endian integers are byte swapped when read; sub-byte + // big-endian bit fields (e.g. IPv4 header fields in lttng-modules net_* events) are still readable. + bool bigEndian = GetString(fieldClass, "byte-order") == "big-endian"; + var length = GetRequiredNumber(fieldClass, "length"); + + var bag = new CtfPropertyBag(); + bag.AddValue("size", length.Text); + bag.AddValue("align", GetAlignment(fieldClass, "alignment").ToString(CultureInfo.InvariantCulture)); + bag.AddValue("signed", signed ? "true" : "false"); + bag.AddValue("byte_order", bigEndian ? "be" : "le"); + bag.AddValue("base", GetDisplayBase(fieldClass).ToString(CultureInfo.InvariantCulture)); + + string map = GetClockMap(fieldClass, context); + if (map != null) + { + bag.AddValue("map", map); + } + + var integer = new CtfIntegerDescriptor(bag); + if (selectorOptions == null) + { + return integer; + } + + // Variant selection in this library is done through an enumeration whose labels are the option names. + var enumeration = new CtfEnumDescriptor(integer); + foreach (var option in selectorOptions) + { + foreach (var range in option.Ranges) + { + var begin = CreateIntegerLiteral(range.Item1, signed); + var end = CreateIntegerLiteral(range.Item2, signed); + if (!enumeration.AddRange(option.Name, new CtfIntegerRange(integer, begin, end))) + { + throw new CtfMetadataException($"Unable to add variant selector range for option '{option.Name}'."); + } + } + } + + return enumeration; + } + + private static CtfMetadataTypeDescriptor BuildFloatingPoint(Dictionary fieldClass) + { + var bag = new CtfPropertyBag(); + switch (GetRequiredNumber(fieldClass, "length").AsLong()) + { + case 32: + bag.AddValue("exp_dig", "8"); + bag.AddValue("mant_dig", "24"); + break; + case 64: + bag.AddValue("exp_dig", "11"); + bag.AddValue("mant_dig", "53"); + break; + default: + throw new CtfMetadataException("Only 32-bit and 64-bit CTF 2 floating point numbers are supported."); + } + + bag.AddValue("align", GetAlignment(fieldClass, "alignment").ToString(CultureInfo.InvariantCulture)); + bag.AddValue("byte_order", GetString(fieldClass, "byte-order") == "big-endian" ? "be" : "le"); + return new CtfFloatingPointDescriptor(bag); + } + + private CtfMetadataTypeDescriptor BuildArrayElement(Dictionary fieldClass, BuildContext context) + { + var element = this.BuildFieldClass(GetRequiredValue(fieldClass, "element-field-class"), context); + if (TryGetNumber(fieldClass, "minimum-alignment", out var minimumAlignment) && minimumAlignment.AsLong() > element.Align) + { + throw new CtfMetadataException("CTF 2 arrays with a minimum alignment greater than their element alignment are not supported."); + } + + return element; + } + + private CtfStructDescriptor BuildStructure(Dictionary fieldClass, BuildContext context) + { + var members = GetValue(fieldClass, "member-classes") as List ?? new List(); + var memberObjects = members.Select(member => member as Dictionary + ?? throw new CtfMetadataException("CTF 2 structure member class is not a JSON object.")).ToList(); + + // A variant selects its option through a sibling integer field; collect the option ranges so that + // the selector can be built as an enumeration. + var selectorOptionsByMember = new Dictionary>(StringComparer.Ordinal); + foreach (var member in memberObjects) + { + var memberClass = this.ResolveFieldClass(GetRequiredValue(member, "field-class")); + if (GetString(memberClass, "type") != "variant") + { + continue; + } + + string selectorName = GetSiblingSelectorName(memberClass); + if (!selectorOptionsByMember.TryGetValue(selectorName, out var selectorOptions)) + { + selectorOptions = new List(); + selectorOptionsByMember.Add(selectorName, selectorOptions); + } + + selectorOptions.AddRange(GetVariantOptions(memberClass, context)); + } + + var fields = new List(memberObjects.Count); + foreach (var member in memberObjects) + { + string memberName = GetRequiredString(member, "name"); + selectorOptionsByMember.TryGetValue(memberName, out var selectorOptions); + var memberType = this.BuildFieldClass(GetRequiredValue(member, "field-class"), context, selectorOptions); + fields.Add(new CtfFieldDescriptor(memberType, context.MapName(memberName))); + } + + var bag = new CtfPropertyBag(); + bag.AddValue("align", GetAlignment(fieldClass, "minimum-alignment").ToString(CultureInfo.InvariantCulture)); + return new CtfStructDescriptor(bag, fields.ToArray()); + } + + private CtfVariantDescriptor BuildVariant(Dictionary fieldClass, BuildContext context) + { + string selectorName = context.MapName(GetSiblingSelectorName(fieldClass)); + + var options = GetValue(fieldClass, "options") as List + ?? throw new CtfMetadataException("CTF 2 variant does not define any options."); + + var fields = new List(options.Count); + for (int index = 0; index < options.Count; index++) + { + var option = options[index] as Dictionary + ?? throw new CtfMetadataException("CTF 2 variant option is not a JSON object."); + var optionType = this.BuildFieldClass(GetRequiredValue(option, "field-class"), context); + fields.Add(new CtfFieldDescriptor(optionType, GetOptionName(option, index, context))); + } + + return new CtfVariantDescriptor(selectorName, fields); + } + + private static IEnumerable GetVariantOptions(Dictionary variantClass, BuildContext context) + { + var options = GetValue(variantClass, "options") as List + ?? throw new CtfMetadataException("CTF 2 variant does not define any options."); + + for (int index = 0; index < options.Count; index++) + { + var option = (Dictionary)options[index]; + var ranges = GetValue(option, "selector-field-ranges") as List + ?? throw new CtfMetadataException("CTF 2 variant option does not define selector field ranges."); + + var optionRanges = new List>(); + foreach (var range in ranges) + { + if (!(range is List bounds) || bounds.Count != 2 || + !(bounds[0] is Ctf2JsonNumber lower) || !(bounds[1] is Ctf2JsonNumber upper)) + { + throw new CtfMetadataException("CTF 2 variant selector field range is invalid."); + } + + optionRanges.Add(Tuple.Create(lower, upper)); + } + + yield return new SelectorOption(GetOptionName(option, index, context), optionRanges); + } + } + + private static string GetOptionName(Dictionary option, int index, BuildContext context) + { + string name = GetString(option, "name"); + return name == null ? $"option{index}" : context.MapName(name); + } + + private static string GetSiblingSelectorName(Dictionary variantClass) + { + var path = GetFieldLocationPath(variantClass, "selector-field-location"); + if (path.Count != 1) + { + throw new CtfMetadataException("Only CTF 2 variants whose selector is a sibling field are supported."); + } + + return path[0]; + } + + private string GetFieldLocation(Dictionary fieldClass, string property, BuildContext context) + { + return string.Join(".", GetFieldLocationPath(fieldClass, property).Select(context.MapName)); + } + + private static List GetFieldLocationPath(Dictionary fieldClass, string property) + { + if (!(GetValue(fieldClass, property) is Dictionary location) || + !(GetValue(location, "path") is List path) || + path.Count == 0) + { + throw new CtfMetadataException($"CTF 2 field class has an invalid '{property}'."); + } + + return path.Select(element => element as string + ?? throw new CtfMetadataException($"CTF 2 '{property}' path elements must be field names.")).ToList(); + } + + private static CtfIntegerDescriptor CreateByte(string encoding, int displayBase) + { + var bag = new CtfPropertyBag(); + bag.AddValue("size", "8"); + bag.AddValue("align", "8"); + bag.AddValue("signed", "false"); + bag.AddValue("byte_order", "le"); + bag.AddValue("base", displayBase.ToString(CultureInfo.InvariantCulture)); + if (encoding != null) + { + bag.AddValue("encoding", encoding); + } + + return new CtfIntegerDescriptor(bag); + } + + private static string GetClockMap(Dictionary fieldClass, BuildContext context) + { + if (context.ClockName == null || !(GetValue(fieldClass, "roles") is List roles)) + { + return null; + } + + bool isTimestamp = roles.OfType().Any(role => + role == DefaultClockTimestampRole || role == PacketEndDefaultClockTimestampRole); + + return isTimestamp ? $"clock.{context.ClockName}.value" : null; + } + + private static int GetDisplayBase(Dictionary fieldClass) + { + return TryGetNumber(fieldClass, "preferred-display-base", out var displayBase) ? (int)displayBase.AsLong() : 10; + } + + private static long GetAlignment(Dictionary fieldClass, string property) + { + return TryGetNumber(fieldClass, property, out var alignment) ? alignment.AsLong() : 1; + } + + private static void EnsureUtf8(Dictionary fieldClass) + { + string encoding = GetString(fieldClass, "encoding"); + if (encoding != null && encoding != "utf-8") + { + throw new CtfMetadataException($"CTF 2 string encoding '{encoding}' is not supported."); + } + } + + private static IntegerLiteral CreateIntegerLiteral(Ctf2JsonNumber number, bool signed) + { + return signed ? new IntegerLiteral(number.AsLong()) : new IntegerLiteral(number.AsULong()); + } + + private static bool TryGetLogLevel(Dictionary eventRecordClass, out string logLevel) + { + logLevel = null; + if (!(GetValue(eventRecordClass, "attributes") is Dictionary attributes)) + { + return false; + } + + foreach (var namespacedAttributes in attributes.Values.OfType>()) + { + if (TryGetNumber(namespacedAttributes, "log-level", out var level) && !level.IsNegative) + { + logLevel = level.Text; + return true; + } + } + + return false; + } + + private static uint GetDataStreamClassId(Dictionary dataStreamClass) + { + return TryGetNumber(dataStreamClass, "id", out var id) ? (uint)id.AsULong() : 0; + } + + private static string FormatUuid(object uuid) + { + if (!(uuid is List bytes) || bytes.Count != 16 || !bytes.All(b => b is Ctf2JsonNumber)) + { + return null; + } + + var hex = string.Concat(bytes.Cast().Select(b => b.AsULong().ToString("x2", CultureInfo.InvariantCulture))); + return $"{hex.Substring(0, 8)}-{hex.Substring(8, 4)}-{hex.Substring(12, 4)}-{hex.Substring(16, 4)}-{hex.Substring(20)}"; + } + + private static object GetValue(Dictionary json, string property) + { + return json.TryGetValue(property, out var value) ? value : null; + } + + private static object GetRequiredValue(Dictionary json, string property) + { + return GetValue(json, property) ?? throw new CtfMetadataException($"CTF 2 metadata is missing the required '{property}' property."); + } + + private static string GetString(Dictionary json, string property) + { + return GetValue(json, property) as string; + } + + private static string GetRequiredString(Dictionary json, string property) + { + return GetString(json, property) ?? throw new CtfMetadataException($"CTF 2 metadata is missing the required '{property}' string property."); + } + + private static bool TryGetNumber(Dictionary json, string property, out Ctf2JsonNumber number) + { + number = GetValue(json, property) as Ctf2JsonNumber; + return number != null; + } + + private static Ctf2JsonNumber GetRequiredNumber(Dictionary json, string property) + { + return TryGetNumber(json, property, out var number) + ? number + : throw new CtfMetadataException($"CTF 2 metadata is missing the required '{property}' number property."); + } + + private sealed class BuildContext + { + internal BuildContext(string clockName, bool prefixNames) + { + this.ClockName = clockName; + this.PrefixNames = prefixNames; + } + + internal string ClockName { get; } + + internal bool PrefixNames { get; } + + internal static BuildContext Header(string clockName) => new BuildContext(clockName, false); + + internal string MapName(string name) => this.PrefixNames ? "_" + name : name; + } + + private sealed class SelectorOption + { + internal SelectorOption(string name, IReadOnlyList> ranges) + { + this.Name = name; + this.Ranges = ranges; + } + + internal string Name { get; } + + internal IReadOnlyList> Ranges { get; } + } + } +} diff --git a/CtfPlayback/Metadata/Ctf2/Ctf2VariableLengthIntegerDescriptor.cs b/CtfPlayback/Metadata/Ctf2/Ctf2VariableLengthIntegerDescriptor.cs new file mode 100644 index 0000000..338770f --- /dev/null +++ b/CtfPlayback/Metadata/Ctf2/Ctf2VariableLengthIntegerDescriptor.cs @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using CtfPlayback.FieldValues; +using CtfPlayback.Helpers; +using CtfPlayback.Metadata.Helpers; +using CtfPlayback.Metadata.TypeInterfaces; +using CtfPlayback.Metadata.Types; + +namespace CtfPlayback.Metadata.Ctf2 +{ + /// + /// CTF 2 variable-length (unsigned or signed LEB128) integer. + /// + internal sealed class Ctf2VariableLengthIntegerDescriptor + : CtfMetadataTypeDescriptor, + ICtfIntegerDescriptor + { + internal Ctf2VariableLengthIntegerDescriptor(bool signed, int displayBase, string map) + : base(CtfTypes.Integer) + { + this.Signed = signed; + this.Base = (short)displayBase; + this.Map = map; + } + + /// + public int Size => 64; + + /// + public bool Signed { get; } + + /// + public string Encoding => "none"; + + /// + public short Base { get; } + + /// + public string Map { get; } + + /// + public override int Align => 8; + + /// + public override CtfFieldValue Read(IPacketReader reader, CtfFieldValue parent = null) + { + Guard.NotNull(reader, nameof(reader)); + + reader.Align((uint)this.Align); + + ulong value = 0; + int shift = 0; + byte current; + do + { + current = reader.ReadBits(8)[0]; + if (shift < 64) + { + value |= (ulong)(current & 0x7f) << shift; + } + + shift += 7; + } + while ((current & 0x80) != 0); + + if (!this.Signed) + { + return new CtfIntegerValue(new IntegerLiteral(value), this); + } + + if (shift < 64 && (current & 0x40) != 0) + { + value |= ulong.MaxValue << shift; + } + + return new CtfIntegerValue(new IntegerLiteral((long)value), this); + } + + /// + public override string ToString() + { + return this.Signed ? "vlint" : "vluint"; + } + } +} diff --git a/CtfPlayback/Metadata/CtfMetadataExtensions.cs b/CtfPlayback/Metadata/CtfMetadataExtensions.cs new file mode 100644 index 0000000..9cbdecf --- /dev/null +++ b/CtfPlayback/Metadata/CtfMetadataExtensions.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using CtfPlayback.Metadata.Interfaces; + +namespace CtfPlayback.Metadata +{ + internal static class CtfMetadataExtensions + { + /// + /// Finds the stream descriptor with the given id. Stream ids are usually also their index, but + /// this is not required (e.g. CTF 2 metadata may declare streams in any order). + /// + internal static ICtfStreamDescriptor GetStream(this ICtfMetadata metadata, uint streamId) + { + var streams = metadata.Streams; + if (streamId < streams.Count && streams[(int)streamId].Id == streamId) + { + return streams[(int)streamId]; + } + + foreach (var stream in streams) + { + if (stream.Id == streamId) + { + return stream; + } + } + + throw new CtfPlaybackException($"The metadata does not describe stream id {streamId}."); + } + } +} diff --git a/CtfPlayback/Metadata/CtfMetadataText.cs b/CtfPlayback/Metadata/CtfMetadataText.cs new file mode 100644 index 0000000..435c52f --- /dev/null +++ b/CtfPlayback/Metadata/CtfMetadataText.cs @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.IO; +using System.Text; + +namespace CtfPlayback.Metadata +{ + /// + /// Helpers for reading a metadata stream, which may be plain text or packetized (see https://diamon.org/ctf/#spec7.1). + /// + internal static class CtfMetadataText + { + private const uint PacketMagic = 0x75d11d57; + + // magic(4) + uuid(16) + checksum(4) + content_size(4) + packet_size(4) + 5 single-byte fields + private const int PacketHeaderSize = 37; + + internal static byte[] ReadAllBytes(Stream stream) + { + using (var memoryStream = new MemoryStream()) + { + stream.CopyTo(memoryStream); + return memoryStream.ToArray(); + } + } + + /// + /// Returns the metadata text, removing packet headers if the metadata is packetized. + /// + internal static string Read(byte[] metadata) + { + if (metadata.Length < PacketHeaderSize || BitConverter.ToUInt32(metadata, 0) != PacketMagic) + { + return Encoding.UTF8.GetString(metadata); + } + + var sb = new StringBuilder(); + int offset = 0; + while (offset + PacketHeaderSize <= metadata.Length) + { + if (BitConverter.ToUInt32(metadata, offset) != PacketMagic) + { + throw new InvalidDataException("Metadata stream seems to be corrupt: invalid packet magic number."); + } + + int contentBytes = (int)(BitConverter.ToUInt32(metadata, offset + 24) / 8); + int packetBytes = (int)(BitConverter.ToUInt32(metadata, offset + 28) / 8); + if (contentBytes < PacketHeaderSize || packetBytes < contentBytes || offset + contentBytes > metadata.Length) + { + throw new InvalidDataException("Metadata stream seems to be corrupt: invalid packet size."); + } + + sb.Append(Encoding.UTF8.GetString(metadata, offset + PacketHeaderSize, contentBytes - PacketHeaderSize)); + offset += packetBytes; + } + + return sb.ToString(); + } + } +} diff --git a/CtfPlayback/Metadata/CtfVersionDetectingMetadataParser.cs b/CtfPlayback/Metadata/CtfVersionDetectingMetadataParser.cs new file mode 100644 index 0000000..82207df --- /dev/null +++ b/CtfPlayback/Metadata/CtfVersionDetectingMetadataParser.cs @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.IO; +using CtfPlayback.Metadata.AntlrParser; +using CtfPlayback.Metadata.Ctf2; +using CtfPlayback.Metadata.Interfaces; + +namespace CtfPlayback.Metadata +{ + /// + /// Parses either CTF 1.8 (TSDL) or CTF 2 (JSON) metadata, based on the content of the metadata stream. + /// + public sealed class CtfVersionDetectingMetadataParser + : ICtfMetadataParser + { + private readonly ICtfMetadataCustomization metadataCustomization; + private readonly ICtfMetadataBuilder metadataBuilder; + private readonly bool prefixCtf2EventFieldNamesWithUnderscore; + + /// + /// Constructor. + /// + /// Extension points for parsing metadata + /// The object used to build metadata + /// + /// See . + /// + public CtfVersionDetectingMetadataParser( + ICtfMetadataCustomization metadataCustomization, + ICtfMetadataBuilder metadataBuilder, + bool prefixCtf2EventFieldNamesWithUnderscore) + { + this.metadataCustomization = metadataCustomization; + this.metadataBuilder = metadataBuilder ?? throw new ArgumentNullException(nameof(metadataBuilder)); + this.prefixCtf2EventFieldNamesWithUnderscore = prefixCtf2EventFieldNamesWithUnderscore; + } + + /// + public ICtfMetadata Parse(Stream metadataStream) + { + if (metadataStream == null) + { + throw new ArgumentNullException(nameof(metadataStream)); + } + + byte[] metadata = CtfMetadataText.ReadAllBytes(metadataStream); + string text = CtfMetadataText.Read(metadata); + + if (Ctf2MetadataParser.IsCtf2Metadata(text)) + { + return new Ctf2MetadataParser(this.metadataBuilder, this.prefixCtf2EventFieldNamesWithUnderscore).Parse(text); + } + + using (var memoryStream = new MemoryStream(metadata, false)) + { + return new CtfAntlrMetadataParser(this.metadataCustomization, this.metadataBuilder).Parse(memoryStream); + } + } + } +} diff --git a/CtfPlayback/Metadata/Types/CtfIntegerDescriptor.cs b/CtfPlayback/Metadata/Types/CtfIntegerDescriptor.cs index b7f5540..ddbc514 100644 --- a/CtfPlayback/Metadata/Types/CtfIntegerDescriptor.cs +++ b/CtfPlayback/Metadata/Types/CtfIntegerDescriptor.cs @@ -62,6 +62,9 @@ internal CtfIntegerDescriptor(CtfPropertyBag bag) internal string ByteOrder { get; private set; } + private bool IsExplicitlyBigEndian => + StringComparer.Ordinal.Equals(this.ByteOrder, "be") || StringComparer.Ordinal.Equals(this.ByteOrder, "network"); + /// public override string ToString() { @@ -98,6 +101,17 @@ internal CtfFieldValue Read(byte[] buffer, int byteCount) // if byte order is not set, or if the value is "native", then endianness is determined by the trace descriptor // if the byte order is "be" or "network", then it is big endian // if the byte order is "le", then it is little endian + // Only explicitly big endian, whole-byte integers are handled for now (e.g. IPv4 addresses in network order). + if (this.IsExplicitlyBigEndian && (this.Size % 8) == 0) + { + var littleEndianBuffer = new byte[byteCount]; + for (int x = 0; x < byteCount; x++) + { + littleEndianBuffer[x] = buffer[byteCount - 1 - x]; + } + + buffer = littleEndianBuffer; + } IntegerLiteral value; diff --git a/CtfUnitTest/Ctf2MetadataTests.cs b/CtfUnitTest/Ctf2MetadataTests.cs new file mode 100644 index 0000000..8e86ae3 --- /dev/null +++ b/CtfUnitTest/Ctf2MetadataTests.cs @@ -0,0 +1,281 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using CtfPlayback; +using CtfPlayback.FieldValues; +using CtfPlayback.Metadata; +using CtfPlayback.Metadata.Ctf2; +using CtfPlayback.Metadata.Interfaces; +using CtfPlayback.Metadata.InternalHelpers; +using CtfPlayback.Metadata.TypeInterfaces; +using CtfPlayback.Metadata.Types; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace CtfUnitTest +{ + [TestClass] + public class Ctf2MetadataTests + { + // Modeled on the metadata written by LTTng 2.16 for a kernel trace with perf counter contexts. + private static readonly string[] LTTngKernelFragments = + { + @"{""type"":""preamble"",""version"":2,""uuid"":[55,65,178,152,114,22,242,74,147,79,201,46,48,147,240,64]}", + @"{""type"":""field-class-alias"",""name"":""u32-erc-id"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":32,""alignment"":8,""byte-order"":""little-endian"",""roles"":[""event-record-class-id""]}}", + @"{""type"":""field-class-alias"",""name"":""u64-ts"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":64,""alignment"":8,""byte-order"":""little-endian"",""roles"":[""default-clock-timestamp""]}}", + @"{""type"":""field-class-alias"",""name"":""er-header-compact"",""field-class"":{""type"":""structure"",""minimum-alignment"":8,""member-classes"":[{""name"":""id"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":5,""byte-order"":""little-endian"",""mappings"":{""compact"":[[0,30]],""extended"":[[31,31]]},""roles"":[""event-record-class-id""]}},{""name"":""v"",""field-class"":{""type"":""variant"",""selector-field-location"":{""path"":[""id""]},""options"":[{""name"":""compact"",""selector-field-ranges"":[[0,30]],""field-class"":{""type"":""structure"",""member-classes"":[{""name"":""timestamp"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":27,""byte-order"":""little-endian"",""roles"":[""default-clock-timestamp""]}}]}},{""name"":""extended"",""selector-field-ranges"":[[31,31]],""field-class"":{""type"":""structure"",""member-classes"":[{""name"":""id"",""field-class"":""u32-erc-id""},{""name"":""timestamp"",""field-class"":""u64-ts""}]}}]}}]}}", + @"{""type"":""field-class-alias"",""name"":""pkt-ctx"",""field-class"":{""type"":""structure"",""member-classes"":[{""name"":""timestamp_begin"",""field-class"":""u64-ts""},{""name"":""timestamp_end"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":64,""alignment"":8,""byte-order"":""little-endian"",""roles"":[""packet-end-default-clock-timestamp""]}},{""name"":""content_size"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":64,""alignment"":8,""byte-order"":""little-endian""}},{""name"":""packet_size"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":64,""alignment"":8,""byte-order"":""little-endian""}},{""name"":""events_discarded"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":64,""alignment"":8,""byte-order"":""little-endian""}},{""name"":""cpu_id"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":32,""alignment"":8,""byte-order"":""little-endian""}}]}}", + @"{""type"":""field-class-alias"",""name"":""pkt-header"",""field-class"":{""type"":""structure"",""member-classes"":[{""name"":""magic"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":32,""alignment"":8,""byte-order"":""little-endian""}},{""name"":""uuid"",""field-class"":{""type"":""static-length-blob"",""length"":16}},{""name"":""stream_id"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":32,""alignment"":8,""byte-order"":""little-endian""}}]}}", + @"{""type"":""trace-class"",""uid"":""3741b298-7216-f24a-934f-c92e3093f040"",""environment"":{""domain"":""kernel"",""tracer_major"":2,""hostname"":""host""},""packet-header-field-class"":""pkt-header""}", + @"{""type"":""clock-class"",""id"":""monotonic"",""name"":""monotonic"",""frequency"":1000000000,""origin"":""unix-epoch"",""offset-from-origin"":{""seconds"":10,""cycles"":5}}", + @"{""type"":""data-stream-class"",""id"":1,""default-clock-class-id"":""monotonic"",""packet-context-field-class"":""pkt-ctx"",""event-record-header-field-class"":""er-header-compact"",""event-record-common-context-field-class"":{""type"":""structure"",""member-classes"":[{""name"":""tid"",""field-class"":{""type"":""fixed-length-signed-integer"",""length"":32,""byte-order"":""little-endian"",""alignment"":8}}]}}", + @"{""type"":""data-stream-class"",""id"":0,""default-clock-class-id"":""monotonic"",""packet-context-field-class"":""pkt-ctx"",""event-record-header-field-class"":""er-header-compact"",""event-record-common-context-field-class"":{""type"":""structure"",""member-classes"":[{""name"":""perf_cpu_cycles"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":64,""byte-order"":""little-endian"",""alignment"":8}},{""name"":""_callstack_kernel_length"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":32,""byte-order"":""little-endian"",""alignment"":8}},{""name"":""callstack_kernel"",""field-class"":{""type"":""dynamic-length-array"",""element-field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":64,""byte-order"":""little-endian"",""alignment"":8,""preferred-display-base"":16},""length-field-location"":{""path"":[""_callstack_kernel_length""]}}}]}}", + @"{""type"":""event-record-class"",""data-stream-class-id"":1,""id"":0,""name"":""sched_wakeup"",""payload-field-class"":{""type"":""structure"",""member-classes"":[{""name"":""comm"",""field-class"":{""type"":""static-length-string"",""length"":16}},{""name"":""tid"",""field-class"":{""type"":""fixed-length-signed-integer"",""length"":32,""byte-order"":""little-endian"",""alignment"":8}}]}}", + @"{""type"":""event-record-class"",""data-stream-class-id"":0,""id"":0,""name"":""sched_switch"",""attributes"":{""lttng.org,2009"":{""log-level"":13}},""payload-field-class"":{""type"":""structure"",""member-classes"":[{""name"":""prev_tid"",""field-class"":{""type"":""fixed-length-signed-integer"",""length"":32,""byte-order"":""little-endian"",""alignment"":8}},{""name"":""filename"",""field-class"":{""type"":""null-terminated-string""}},{""name"":""address_ipv4"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":32,""byte-order"":""big-endian"",""alignment"":8,""preferred-display-base"":16}},{""name"":""ihl"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":4,""byte-order"":""big-endian""}},{""name"":""ratio"",""field-class"":{""type"":""fixed-length-floating-point-number"",""length"":64,""byte-order"":""little-endian"",""alignment"":8}}]}}", + }; + + private static string LTTngKernelMetadata => string.Concat(LTTngKernelFragments.Select(fragment => "\u001e" + fragment + "\n")); + + [TestMethod] + public void DetectsCtf2Metadata() + { + Assert.IsTrue(Ctf2MetadataParser.IsCtf2Metadata(LTTngKernelMetadata)); + Assert.IsTrue(Ctf2MetadataParser.IsCtf2Metadata("\n \u001e{}")); + Assert.IsFalse(Ctf2MetadataParser.IsCtf2Metadata("/* CTF 1.8 */ trace { major = 1; };")); + Assert.IsFalse(Ctf2MetadataParser.IsCtf2Metadata(string.Empty)); + } + + [TestMethod] + public void ParsesTraceClockAndStreams() + { + var builder = Parse(LTTngKernelMetadata); + + Assert.AreEqual(Guid.Parse("3741b298-7216-f24a-934f-c92e3093f040"), builder.TraceDescriptor.Uuid); + Assert.AreEqual("kernel", builder.EnvironmentDescriptor.Properties["domain"]); + Assert.AreEqual("2", builder.EnvironmentDescriptor.Properties["tracer_major"]); + CollectionAssert.AreEqual( + new[] { "magic", "uuid", "stream_id" }, + builder.TraceDescriptor.PacketHeader.Fields.Select(f => f.Name).ToArray()); + + var clock = builder.ClocksByName["monotonic"]; + Assert.AreEqual(1000000000ul, clock.Frequency); + Assert.AreEqual(10000000005ul, clock.Offset); + + // Streams are declared out of order in the metadata but are stored by id. + CollectionAssert.AreEqual(new uint[] { 0, 1 }, builder.Streams.Select(s => s.Id).ToArray()); + } + + [TestMethod] + public void EventHeaderVariantSelectorIsAnEnumeration() + { + var builder = Parse(LTTngKernelMetadata); + var header = builder.Streams[0].EventHeader; + + var id = header.GetField("id").TypeDescriptor as ICtfEnumDescriptor; + Assert.IsNotNull(id); + Assert.AreEqual("compact", id.GetName(5ul)); + Assert.AreEqual("extended", id.GetName(31ul)); + + var variant = header.GetField("v").TypeDescriptor as ICtfVariantDescriptor; + Assert.IsNotNull(variant); + Assert.AreEqual("id", variant.Switch); + + var compactTimestamp = ((ICtfStructDescriptor)variant.GetVariant("compact").TypeDescriptor).GetField("timestamp").TypeDescriptor as ICtfIntegerDescriptor; + Assert.AreEqual(27, compactTimestamp.Size); + Assert.AreEqual("clock.monotonic.value", compactTimestamp.Map); + } + + [TestMethod] + public void EventFieldNamesArePrefixedLikeCtf18() + { + var builder = Parse(LTTngKernelMetadata); + + var context = builder.Streams[0].EventContext; + CollectionAssert.AreEqual( + new[] { "_perf_cpu_cycles", "__callstack_kernel_length", "_callstack_kernel" }, + context.Fields.Select(f => f.Name).ToArray()); + + var callstack = context.GetField("_callstack_kernel").TypeDescriptor as ICtfArrayDescriptor; + Assert.AreEqual("__callstack_kernel_length", callstack.Index); + + // Header and packet scopes keep their names. + Assert.IsNotNull(builder.Streams[0].PacketContext.GetField("timestamp_begin")); + } + + [TestMethod] + public void EventsAreIdentifiedByStreamAndId() + { + var builder = Parse(LTTngKernelMetadata); + + Assert.AreEqual(2, builder.AddedEvents.Count); + + var sched_switch = builder.AddedEvents.Single(e => e.Assignments["name"] == "sched_switch"); + Assert.AreEqual("0", sched_switch.Assignments["stream_id"]); + Assert.AreEqual("0", sched_switch.Assignments["id"]); + Assert.AreEqual("13", sched_switch.Assignments["loglevel"]); + + var sched_wakeup = builder.AddedEvents.Single(e => e.Assignments["name"] == "sched_wakeup"); + Assert.AreEqual("1", sched_wakeup.Assignments["stream_id"]); + Assert.AreEqual("0", sched_wakeup.Assignments["id"]); + + var payload = (ICtfStructDescriptor)sched_wakeup.TypeDeclarations["fields"]; + var comm = payload.GetField("_comm").TypeDescriptor as ICtfArrayDescriptor; + Assert.IsNotNull(comm); + Assert.AreEqual("16", comm.Index); + Assert.AreEqual("UTF8", ((ICtfIntegerDescriptor)comm.Type).Encoding); + + var switchPayload = (ICtfStructDescriptor)sched_switch.TypeDeclarations["fields"]; + Assert.IsInstanceOfType(switchPayload.GetField("_filename").TypeDescriptor, typeof(ICtfStringDescriptor)); + Assert.AreEqual(4, ((ICtfIntegerDescriptor)switchPayload.GetField("_ihl").TypeDescriptor).Size); + + var ratio = switchPayload.GetField("_ratio").TypeDescriptor as ICtfFloatingPointDescriptor; + Assert.IsNotNull(ratio); + Assert.AreEqual(11, ratio.Exponent); + Assert.AreEqual(53, ratio.Mantissa); + } + + [TestMethod] + public void VersionDetectingParserHandlesPacketizedCtf2Metadata() + { + byte[] text = Encoding.UTF8.GetBytes(LTTngKernelMetadata); + var stream = new MemoryStream(); + var writer = new BinaryWriter(stream); + writer.Write(0x75d11d57u); + writer.Write(new byte[16]); + writer.Write(0u); + writer.Write((uint)((37 + text.Length) * 8)); + writer.Write((uint)((37 + text.Length) * 8)); + writer.Write(new byte[] { 0, 0, 0, 2, 0 }); + writer.Write(text); + stream.Position = 0; + + var builder = new Ctf2TestMetadataBuilder(); + new CtfVersionDetectingMetadataParser(null, builder, true).Parse(stream); + + Assert.AreEqual(2, builder.Streams.Count); + Assert.AreEqual(2, builder.AddedEvents.Count); + } + + [TestMethod] + public void RejectsUnsupportedVersion() + { + Assert.ThrowsException(() => Parse("\u001e{\"type\":\"preamble\",\"version\":3}")); + } + + [TestMethod] + public void ReadsBigEndianIntegers() + { + var builder = Parse(LTTngKernelMetadata); + var sched_switch = builder.AddedEvents.Single(e => e.Assignments["name"] == "sched_switch"); + var address = (CtfIntegerDescriptor)((ICtfStructDescriptor)sched_switch.TypeDeclarations["fields"]).GetField("_address_ipv4").TypeDescriptor; + + var value = (CtfIntegerValue)address.Read(new byte[] { 127, 0, 0, 1 }, 4); + Assert.AreEqual(0x7F000001ul, value.Value.ValueAsUlong); + } + + [TestMethod] + public void ReadsVariableLengthIntegers() + { + var unsigned = new Ctf2VariableLengthIntegerDescriptor(false, 10, null); + var signed = new Ctf2VariableLengthIntegerDescriptor(true, 10, null); + + Assert.AreEqual(624485ul, ((CtfIntegerValue)unsigned.Read(new BytePacketReader(0xE5, 0x8E, 0x26))).Value.ValueAsUlong); + Assert.AreEqual(-123456L, ((CtfIntegerValue)signed.Read(new BytePacketReader(0xC0, 0xBB, 0x78))).Value.ValueAsLong); + } + + [TestMethod] + public void JsonReaderHandlesLargeNumbersAndEscapes() + { + var json = (Dictionary)Ctf2Json.Parse(@"{ ""a"": 18446744073709551615, ""b"": ""x\""\u0041\n"", ""c"": [true, false, null, -5] }"); + + Assert.AreEqual(ulong.MaxValue, ((Ctf2JsonNumber)json["a"]).AsULong()); + Assert.AreEqual("x\"A\n", json["b"]); + var array = (List)json["c"]; + Assert.AreEqual(true, array[0]); + Assert.AreEqual(false, array[1]); + Assert.IsNull(array[2]); + Assert.AreEqual(-5L, ((Ctf2JsonNumber)array[3]).AsLong()); + + Assert.ThrowsException(() => Ctf2Json.Parse("{\"a\": }")); + } + + private static Ctf2TestMetadataBuilder Parse(string metadata) + { + var builder = new Ctf2TestMetadataBuilder(); + new Ctf2MetadataParser(builder, prefixEventFieldNamesWithUnderscore: true).Parse(metadata); + return builder; + } + + private sealed class AddedEvent + { + public IReadOnlyDictionary Assignments { get; set; } + + public IReadOnlyDictionary TypeDeclarations { get; set; } + } + + private sealed class Ctf2TestMetadataBuilder + : ICtfMetadataBuilder + { + private readonly List clocks = new List(); + private readonly List streams = new List(); + + public List AddedEvents { get; } = new List(); + + public ICtfTraceDescriptor TraceDescriptor { get; private set; } + + public ICtfEnvironmentDescriptor EnvironmentDescriptor { get; private set; } + + public IReadOnlyList Clocks => this.clocks; + + public IReadOnlyDictionary ClocksByName => this.clocks.ToDictionary(c => c.Name); + + public IReadOnlyList Streams => this.streams; + + public IReadOnlyList Events => Array.Empty(); + + public void SetTraceDescriptor(ICtfTraceDescriptor traceDescriptor) => this.TraceDescriptor = traceDescriptor; + + public void SetEnvironmentDescriptor(ICtfEnvironmentDescriptor environmentDescriptor) => this.EnvironmentDescriptor = environmentDescriptor; + + public void AddEvent(IReadOnlyDictionary assignments, IReadOnlyDictionary typeDeclarations) + { + this.AddedEvents.Add(new AddedEvent { Assignments = assignments, TypeDeclarations = typeDeclarations }); + } + + public void AddClock(ICtfClockDescriptor clockDescriptor) => this.clocks.Add(clockDescriptor); + + public void AddStream(ICtfStreamDescriptor streamDescriptor) => this.streams.Add(streamDescriptor); + } + + private sealed class BytePacketReader + : IPacketReader + { + private readonly Queue bytes; + + public BytePacketReader(params byte[] bytes) + { + this.bytes = new Queue(bytes); + } + + public bool EndOfStream => this.bytes.Count == 0; + + public uint RemainingBufferedBitCount => (uint)this.bytes.Count * 8; + + public byte[] ReadBits(uint bitCount) + { + Assert.AreEqual(8u, bitCount); + return new[] { this.bytes.Dequeue() }; + } + + public byte[] ReadString() => throw new NotSupportedException(); + + public void Align(uint bitCount) + { + } + } + } +} diff --git a/LTTngCds/CookerData/LTTngEvent.cs b/LTTngCds/CookerData/LTTngEvent.cs index cb8f5ad..e3de897 100644 --- a/LTTngCds/CookerData/LTTngEvent.cs +++ b/LTTngCds/CookerData/LTTngEvent.cs @@ -50,10 +50,15 @@ internal LTTngEvent(LTTngEvent lttngEvent, long timestampAdjustmentNano) : this( } /// - /// Event Id + /// Event Id. Event ids are only unique within a stream, see . /// public uint Id => this.eventDescriptor.Id; + /// + /// Id of the stream (LTTng channel) that contains the event. + /// + public uint StreamId => this.ctfEvent.StreamId; + /// /// Event name /// diff --git a/LTTngCds/CtfExtensions/Descriptors/EventDescriptor.cs b/LTTngCds/CtfExtensions/Descriptors/EventDescriptor.cs index 4c7d37f..d7f8923 100644 --- a/LTTngCds/CtfExtensions/Descriptors/EventDescriptor.cs +++ b/LTTngCds/CtfExtensions/Descriptors/EventDescriptor.cs @@ -31,6 +31,11 @@ internal EventDescriptor( this.Name = string.Intern(name.Trim(new[] { '"' })); this.SetPayload(typeDeclarations); + + if (typeDeclarations.TryGetValue("context", out var context)) + { + this.Context = context; + } } public uint Id { get; private set; } @@ -41,9 +46,9 @@ internal EventDescriptor( public uint LogLevel { get; private set; } - // LTTng defines no context for an event + // CTF 1.8 LTTng traces define no context for an event; CTF 2 traces may define a specific context. - public ICtfTypeDescriptor Context => null; + public ICtfTypeDescriptor Context { get; } public ICtfTypeDescriptor Payload { get; private set; } diff --git a/LTTngCds/CtfExtensions/FolderInput/LTTngFolderInput.cs b/LTTngCds/CtfExtensions/FolderInput/LTTngFolderInput.cs index 5e7ddbb..576d920 100644 --- a/LTTngCds/CtfExtensions/FolderInput/LTTngFolderInput.cs +++ b/LTTngCds/CtfExtensions/FolderInput/LTTngFolderInput.cs @@ -35,7 +35,7 @@ public LTTngFolderInput(string folderPath) Debug.Assert(traceDirectoryPath != null, nameof(traceDirectoryPath) + " != null"); var associatedEntries = Directory.GetFiles(traceDirectoryPath).Where(entry => - Path.GetFileName(entry).StartsWith("chan")); + LTTngStreamFiles.IsEventStreamFile(Path.GetFileName(entry))); traceInput.EventStreams = associatedEntries.Select( fileName => new LTTngFileInputStream(fileName)).Cast().ToList(); diff --git a/LTTngCds/CtfExtensions/LTTngMetadata.cs b/LTTngCds/CtfExtensions/LTTngMetadata.cs index 49ffa4d..78bd29b 100644 --- a/LTTngCds/CtfExtensions/LTTngMetadata.cs +++ b/LTTngCds/CtfExtensions/LTTngMetadata.cs @@ -15,7 +15,7 @@ internal class LTTngMetadata private readonly List streams = new List(); private readonly List clocks = new List(); private readonly List events = new List(); - private readonly Dictionary eventsById = new Dictionary(); + private readonly Dictionary<(int StreamId, uint EventId), ICtfEventDescriptor> eventsByStreamAndId = new Dictionary<(int StreamId, uint EventId), ICtfEventDescriptor>(); private readonly Dictionary clocksByName = new Dictionary(); public ICtfTraceDescriptor TraceDescriptor { get; private set; } @@ -30,7 +30,13 @@ internal class LTTngMetadata public IReadOnlyList Events => this.events; - public IReadOnlyDictionary EventByEventId => this.eventsById; + /// + /// Event ids are only unique within a stream (e.g. with multiple channels, or in CTF 2 traces). + /// + public bool TryGetEvent(uint streamId, uint eventId, out ICtfEventDescriptor eventDescriptor) + { + return this.eventsByStreamAndId.TryGetValue(((int)streamId, eventId), out eventDescriptor); + } public void SetTraceDescriptor(ICtfTraceDescriptor traceDescriptor) { @@ -52,7 +58,7 @@ public void AddEvent( { var eventDescriptor = new EventDescriptor(assignments, typeDeclarations); this.events.Add(eventDescriptor); - this.eventsById.Add(eventDescriptor.Id, eventDescriptor); + this.eventsByStreamAndId.Add((eventDescriptor.Stream, eventDescriptor.Id), eventDescriptor); } public void AddClock(ICtfClockDescriptor clockDescriptor) diff --git a/LTTngCds/CtfExtensions/LTTngPlaybackCustomization.cs b/LTTngCds/CtfExtensions/LTTngPlaybackCustomization.cs index 0e50e8b..2628802 100644 --- a/LTTngCds/CtfExtensions/LTTngPlaybackCustomization.cs +++ b/LTTngCds/CtfExtensions/LTTngPlaybackCustomization.cs @@ -10,7 +10,6 @@ using CtfPlayback.FieldValues; using CtfPlayback.Inputs; using CtfPlayback.Metadata; -using CtfPlayback.Metadata.AntlrParser; using CtfPlayback.Metadata.Interfaces; using LTTngCds.CookerData; using LTTngCds.CtfExtensions.Descriptors; @@ -41,7 +40,10 @@ public virtual ICtfMetadataParser CreateMetadataParser( ICtfTraceInput traceInput) { this.metadataCustomization.PrepareForNewTrace(traceInput.MetadataStream); - return new CtfAntlrMetadataParser(this.metadataCustomization, this.metadataCustomization.LTTngMetadata); + return new CtfVersionDetectingMetadataParser( + this.metadataCustomization, + this.metadataCustomization.LTTngMetadata, + prefixCtf2EventFieldNamesWithUnderscore: true); } public bool GetTimestampsFromPacketContext( @@ -223,22 +225,24 @@ public ulong GetPacketContentBitCount(ICtfPacket ctfPacket) public ICtfEventDescriptor GetEventDescriptor(ICtfEvent ctfEvent, ICtfMetadata metadata) { uint id = this.GetEventId(ctfEvent); + uint streamId = ctfEvent.StreamId; ICtfEventDescriptor eventDescriptor; if (metadata is LTTngMetadata typedMetadata) { // optimization if we got our own typed metadata back // we check for success below - typedMetadata.EventByEventId.TryGetValue(id, out eventDescriptor); + typedMetadata.TryGetEvent(streamId, id, out eventDescriptor); } else { - eventDescriptor = metadata.Events.FirstOrDefault(x => x.Id == id); + eventDescriptor = metadata.Events.FirstOrDefault(x => + x.Id == id && (!(x is EventDescriptor lttngDescriptor) || lttngDescriptor.Stream == streamId)); } if (eventDescriptor == null) { - throw new LTTngPlaybackException($"Unable to find event descriptor for event id={id}."); + throw new LTTngPlaybackException($"Unable to find event descriptor for stream id={streamId}, event id={id}."); } return eventDescriptor; diff --git a/LTTngCds/CtfExtensions/LTTngStreamFiles.cs b/LTTngCds/CtfExtensions/LTTngStreamFiles.cs new file mode 100644 index 0000000..80efbb2 --- /dev/null +++ b/LTTngCds/CtfExtensions/LTTngStreamFiles.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; + +namespace LTTngCds.CtfExtensions +{ + internal static class LTTngStreamFiles + { + /// + /// LTTng names event stream files "<channel name>_<cpu>", e.g. "channel0_3" or "sw-chan_3". + /// + /// File name without directory + /// true if the file is an event stream + internal static bool IsEventStreamFile(string fileName) + { + if (string.IsNullOrEmpty(fileName) || + fileName.StartsWith(".", StringComparison.Ordinal) || + StringComparer.Ordinal.Equals(fileName, "metadata")) + { + return false; + } + + int separatorIndex = fileName.LastIndexOf('_'); + if (separatorIndex <= 0 || separatorIndex == fileName.Length - 1) + { + return false; + } + + for (int index = separatorIndex + 1; index < fileName.Length; index++) + { + if (!char.IsDigit(fileName[index])) + { + return false; + } + } + + return true; + } + } +} diff --git a/LTTngCds/CtfExtensions/ZipArchiveInput/LTTngZipArchiveInput.cs b/LTTngCds/CtfExtensions/ZipArchiveInput/LTTngZipArchiveInput.cs index b843252..c017780 100644 --- a/LTTngCds/CtfExtensions/ZipArchiveInput/LTTngZipArchiveInput.cs +++ b/LTTngCds/CtfExtensions/ZipArchiveInput/LTTngZipArchiveInput.cs @@ -39,7 +39,7 @@ public LTTngZipArchiveInput(ZipArchive archive) var associatedArchiveEntries = archive.Entries.Where(entry => Path.GetDirectoryName(entry.FullName) == traceDirectoryPath && - Path.GetFileName(entry.FullName).StartsWith("chan")); + LTTngStreamFiles.IsEventStreamFile(Path.GetFileName(entry.FullName))); traceInput.EventStreams = associatedArchiveEntries.Select( archiveEntry => new LTTngZipArchiveInputStream(archiveEntry)).Cast().ToList(); diff --git a/LTTngDataExtUnitTest/LTTngCtf2UnitTest.cs b/LTTngDataExtUnitTest/LTTngCtf2UnitTest.cs new file mode 100644 index 0000000..100cb5a --- /dev/null +++ b/LTTngDataExtUnitTest/LTTngCtf2UnitTest.cs @@ -0,0 +1,221 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text; +using LTTngDataExtensions.DataOutputTypes; +using LTTngDataExtensions.SourceDataCookers; +using LTTngDataExtensions.SourceDataCookers.Thread; +using LTTngDataExtensions.Tables; +using Microsoft.Performance.SDK.Extensibility; +using Microsoft.Performance.SDK.Processing; +using Microsoft.Performance.Toolkit.Engine; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace LTTngDataExtUnitTest +{ + /// + /// Processes a small synthetic LTTng kernel trace written in the CTF 2 format, with channels that are not named + /// "chan*" and event ids that are only unique within a channel (as written by LTTng 2.15+). + /// + [TestClass] + public class LTTngCtf2UnitTest + { + private const string Metadata = + "\u001e{\"type\":\"preamble\",\"version\":2}\n" + + "\u001e{\"type\":\"field-class-alias\",\"name\":\"u64-ts\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":64,\"alignment\":8,\"byte-order\":\"little-endian\",\"roles\":[\"default-clock-timestamp\"]}}\n" + + "\u001e{\"type\":\"field-class-alias\",\"name\":\"u64\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":64,\"alignment\":8,\"byte-order\":\"little-endian\"}}\n" + + "\u001e{\"type\":\"field-class-alias\",\"name\":\"u32\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":32,\"alignment\":8,\"byte-order\":\"little-endian\"}}\n" + + "\u001e{\"type\":\"field-class-alias\",\"name\":\"s32\",\"field-class\":{\"type\":\"fixed-length-signed-integer\",\"length\":32,\"alignment\":8,\"byte-order\":\"little-endian\"}}\n" + + "\u001e{\"type\":\"field-class-alias\",\"name\":\"comm\",\"field-class\":{\"type\":\"static-length-string\",\"length\":16}}\n" + + "\u001e{\"type\":\"field-class-alias\",\"name\":\"er-header-compact\",\"field-class\":{\"type\":\"structure\",\"minimum-alignment\":8,\"member-classes\":[{\"name\":\"id\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":5,\"byte-order\":\"little-endian\",\"mappings\":{\"compact\":[[0,30]],\"extended\":[[31,31]]}}},{\"name\":\"v\",\"field-class\":{\"type\":\"variant\",\"selector-field-location\":{\"path\":[\"id\"]},\"options\":[{\"name\":\"compact\",\"selector-field-ranges\":[[0,30]],\"field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"timestamp\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":27,\"byte-order\":\"little-endian\",\"roles\":[\"default-clock-timestamp\"]}}]}},{\"name\":\"extended\",\"selector-field-ranges\":[[31,31]],\"field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"id\",\"field-class\":\"u32\"},{\"name\":\"timestamp\",\"field-class\":\"u64-ts\"}]}}]}}]}}\n" + + "\u001e{\"type\":\"field-class-alias\",\"name\":\"pkt-ctx\",\"field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"timestamp_begin\",\"field-class\":\"u64-ts\"},{\"name\":\"timestamp_end\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":64,\"alignment\":8,\"byte-order\":\"little-endian\",\"roles\":[\"packet-end-default-clock-timestamp\"]}},{\"name\":\"content_size\",\"field-class\":\"u64\"},{\"name\":\"packet_size\",\"field-class\":\"u64\"},{\"name\":\"events_discarded\",\"field-class\":\"u64\"},{\"name\":\"cpu_id\",\"field-class\":\"u32\"}]}}\n" + + "\u001e{\"type\":\"field-class-alias\",\"name\":\"pkt-header\",\"field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"magic\",\"field-class\":\"u32\"},{\"name\":\"uuid\",\"field-class\":{\"type\":\"static-length-blob\",\"length\":16}},{\"name\":\"stream_id\",\"field-class\":\"u32\"}]}}\n" + + "\u001e{\"type\":\"trace-class\",\"environment\":{\"domain\":\"kernel\",\"tracer_name\":\"lttng-modules\",\"tracer_major\":2,\"tracer_minor\":16},\"packet-header-field-class\":\"pkt-header\"}\n" + + "\u001e{\"type\":\"clock-class\",\"id\":\"monotonic\",\"name\":\"monotonic\",\"frequency\":1000000000,\"origin\":\"unix-epoch\",\"offset-from-origin\":{\"seconds\":1790000000,\"cycles\":0}}\n" + + "\u001e{\"type\":\"data-stream-class\",\"id\":1,\"default-clock-class-id\":\"monotonic\",\"packet-context-field-class\":\"pkt-ctx\",\"event-record-header-field-class\":\"er-header-compact\",\"event-record-common-context-field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"pid\",\"field-class\":\"s32\"},{\"name\":\"tid\",\"field-class\":\"s32\"},{\"name\":\"procname\",\"field-class\":\"comm\"},{\"name\":\"prio\",\"field-class\":\"s32\"}]}}\n" + + "\u001e{\"type\":\"data-stream-class\",\"id\":0,\"default-clock-class-id\":\"monotonic\",\"packet-context-field-class\":\"pkt-ctx\",\"event-record-header-field-class\":\"er-header-compact\",\"event-record-common-context-field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"perf_cpu_cycles\",\"field-class\":\"u64\"},{\"name\":\"perf_cpu_instructions\",\"field-class\":\"u64\"},{\"name\":\"pid\",\"field-class\":\"s32\"},{\"name\":\"tid\",\"field-class\":\"s32\"},{\"name\":\"procname\",\"field-class\":\"comm\"},{\"name\":\"prio\",\"field-class\":\"s32\"}]}}\n" + + "\u001e{\"type\":\"event-record-class\",\"data-stream-class-id\":1,\"id\":0,\"name\":\"sched_wakeup\",\"payload-field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"comm\",\"field-class\":\"comm\"},{\"name\":\"tid\",\"field-class\":\"s32\"},{\"name\":\"prio\",\"field-class\":\"s32\"},{\"name\":\"target_cpu\",\"field-class\":\"s32\"}]}}\n" + + "\u001e{\"type\":\"event-record-class\",\"data-stream-class-id\":0,\"id\":0,\"name\":\"sched_switch\",\"payload-field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"prev_comm\",\"field-class\":\"comm\"},{\"name\":\"prev_tid\",\"field-class\":\"s32\"},{\"name\":\"prev_prio\",\"field-class\":\"s32\"},{\"name\":\"prev_state\",\"field-class\":{\"type\":\"fixed-length-signed-integer\",\"length\":64,\"alignment\":8,\"byte-order\":\"little-endian\"}},{\"name\":\"next_comm\",\"field-class\":\"comm\"},{\"name\":\"next_tid\",\"field-class\":\"s32\"},{\"name\":\"next_prio\",\"field-class\":\"s32\"}]}}\n"; + + private static string tracePath; + private static RuntimeExecutionResults results; + private static DataCookerPath threadCookerPath; + private static DataCookerPath genericCookerPath; + + [ClassInitialize] + public static void ProcessTrace(TestContext context) + { + tracePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".ctf"); + WriteTrace(tracePath); + + var runtime = Engine.Create(new FileDataSource(tracePath)); + + var threadCooker = new LTTngThreadDataCooker(); + threadCookerPath = threadCooker.Path; + runtime.EnableCooker(threadCookerPath); + + var genericCooker = new LTTngGenericEventDataCooker(); + genericCookerPath = genericCooker.Path; + runtime.EnableCooker(genericCookerPath); + + runtime.EnableTable(ExecutionEventTable.TableDescriptor); + + results = runtime.Process(); + } + + [ClassCleanup] + public static void Cleanup() + { + if (tracePath != null && File.Exists(tracePath)) + { + File.Delete(tracePath); + } + } + + [TestMethod] + public void ProcessesAllChannels() + { + Assert.AreEqual(0, results.ProcessingErrors.Count(), string.Join(Environment.NewLine, results.ProcessingErrors.Select(e => e.ProcessFault?.ToString()))); + + var events = results.QueryOutput>( + new DataOutputPath(genericCookerPath, nameof(LTTngGenericEventDataCooker.Events))); + + var names = Enumerable.Range(0, (int)events.Count).Select(i => events[i].EventName).ToList(); + Assert.AreEqual(3, names.Count(n => n == "sched_switch")); + Assert.AreEqual(1, names.Count(n => n == "sched_wakeup")); + + var wakeup = Enumerable.Range(0, (int)events.Count).Select(i => events[i]).Single(e => e.EventName == "sched_wakeup"); + Assert.AreEqual("taskA", wakeup.FieldValues[wakeup.FieldNames.IndexOf("_comm")]); + } + + [TestMethod] + public void ExecutionEventsHavePerformanceCounterDeltas() + { + var executionEvents = results.QueryOutput>( + new DataOutputPath(threadCookerPath, nameof(LTTngThreadDataCooker.ExecutionEvents))); + + Assert.AreEqual(2, executionEvents.Count); + + var taskA = executionEvents.Single(e => e.NextTid == 100); + Assert.AreEqual("taskA", taskA.NextImage); + Assert.AreEqual(500, taskA.PerformanceCountersDiffByName["_perf_cpu_cycles"]); + Assert.AreEqual(600, taskA.PerformanceCountersDiffByName["_perf_cpu_instructions"]); + + var idle = executionEvents.Single(e => e.NextTid == 0); + Assert.AreEqual(300, idle.PerformanceCountersDiffByName["_perf_cpu_cycles"]); + Assert.AreEqual(400, idle.PerformanceCountersDiffByName["_perf_cpu_instructions"]); + + var table = results.BuildTable(ExecutionEventTable.TableDescriptor); + var cycles = table.Columns.Single(c => c.Configuration.Metadata.Name == "CPU Cycle"); + var instructions = table.Columns.Single(c => c.Configuration.Metadata.Name == "Instruction Count"); + CollectionAssert.AreEquivalent(new long[] { 500, 300 }, Enumerable.Range(0, table.RowCount).Select(i => (long)cycles.Project(i)).ToArray()); + CollectionAssert.AreEquivalent(new long[] { 600, 400 }, Enumerable.Range(0, table.RowCount).Select(i => (long)instructions.Project(i)).ToArray()); + } + + private static void WriteTrace(string path) + { + using (var archive = ZipFile.Open(path, ZipArchiveMode.Create)) + { + WriteEntry(archive, "trace/kernel/metadata", Encoding.UTF8.GetBytes(Metadata)); + + // Channel "sw-chan" (stream class 0) on CPU 0: idle -> taskA -> idle -> taskA. + WriteEntry(archive, "trace/kernel/sw-chan_0", Packet(0, 1000, 3000, + SchedSwitch(1000, cycles: 1000, instructions: 2000, prevTid: 0, prevComm: "swapper/0", nextTid: 100, nextComm: "taskA"), + SchedSwitch(2000, cycles: 1500, instructions: 2600, prevTid: 100, prevComm: "taskA", nextTid: 0, nextComm: "swapper/0"), + SchedSwitch(3000, cycles: 1800, instructions: 3000, prevTid: 0, prevComm: "swapper/0", nextTid: 100, nextComm: "taskA"))); + + // Channel "other-chan" (stream class 1) on CPU 0: its event id 0 is sched_wakeup, not sched_switch. + WriteEntry(archive, "trace/kernel/other-chan_0", Packet(1, 1500, 1500, SchedWakeup(1500, tid: 100, comm: "taskA"))); + } + } + + private static void WriteEntry(ZipArchive archive, string name, byte[] content) + { + using (var stream = archive.CreateEntry(name).Open()) + { + stream.Write(content, 0, content.Length); + } + } + + private static byte[] Packet(uint streamId, ulong begin, ulong end, params byte[][] events) + { + const int headerAndContextSize = 4 + 16 + 4 + (5 * 8) + 4; + ulong sizeInBits = (ulong)(headerAndContextSize + events.Sum(e => e.Length)) * 8; + + var stream = new MemoryStream(); + var writer = new BinaryWriter(stream); + writer.Write(0xC1FC1FC1u); + writer.Write(new byte[16]); + writer.Write(streamId); + writer.Write(begin); + writer.Write(end); + writer.Write(sizeInBits); + writer.Write(sizeInBits); + writer.Write(0ul); + writer.Write(0u); + foreach (var e in events) + { + writer.Write(e); + } + + return stream.ToArray(); + } + + private static byte[] SchedSwitch(uint timestamp, ulong cycles, ulong instructions, int prevTid, string prevComm, int nextTid, string nextComm) + { + var stream = new MemoryStream(); + var writer = new BinaryWriter(stream); + WriteCompactHeader(writer, 0, timestamp); + writer.Write(cycles); + writer.Write(instructions); + WriteProcessContext(writer, prevTid, prevComm); + writer.Write(Comm(prevComm)); + writer.Write(prevTid); + writer.Write(20); + writer.Write(0L); + writer.Write(Comm(nextComm)); + writer.Write(nextTid); + writer.Write(20); + return stream.ToArray(); + } + + private static byte[] SchedWakeup(uint timestamp, int tid, string comm) + { + var stream = new MemoryStream(); + var writer = new BinaryWriter(stream); + WriteCompactHeader(writer, 0, timestamp); + WriteProcessContext(writer, 0, "swapper/0"); + writer.Write(Comm(comm)); + writer.Write(tid); + writer.Write(20); + writer.Write(0); + return stream.ToArray(); + } + + private static void WriteCompactHeader(BinaryWriter writer, uint id, uint timestamp) + { + // 5-bit event id followed by a 27-bit timestamp, packed least significant bit first. + writer.Write(id | (timestamp << 5)); + } + + private static void WriteProcessContext(BinaryWriter writer, int tid, string procname) + { + writer.Write(tid); + writer.Write(tid); + writer.Write(Comm(procname)); + writer.Write(20); + } + + private static byte[] Comm(string value) + { + var bytes = new byte[16]; + Encoding.UTF8.GetBytes(value, 0, value.Length, bytes, 0); + return bytes; + } + } +} diff --git a/LTTngDataExtensions/DataOutputTypes/LTTngGenericEvent.cs b/LTTngDataExtensions/DataOutputTypes/LTTngGenericEvent.cs index 3b46c1d..319414d 100644 --- a/LTTngDataExtensions/DataOutputTypes/LTTngGenericEvent.cs +++ b/LTTngDataExtensions/DataOutputTypes/LTTngGenericEvent.cs @@ -16,11 +16,13 @@ public class EventKind private class Key { private string domain; + private uint streamId; private uint id; - public Key(string domain, uint id) + public Key(string domain, uint streamId, uint id) { this.domain = domain ?? string.Empty; + this.streamId = streamId; this.id = id; } @@ -28,7 +30,7 @@ public override bool Equals(object obj) { if (obj is Key key) { - return domain.Equals(key.domain) && id.Equals(key.id); + return domain.Equals(key.domain) && streamId.Equals(key.streamId) && id.Equals(key.id); } else { @@ -40,7 +42,8 @@ public override int GetHashCode() { int h1 = domain.GetHashCode(); int h2 = id.GetHashCode(); - return ((h1 << 5) + h1) ^ h2; + int h3 = streamId.GetHashCode(); + return ((((h1 << 5) + h1) ^ h2) * 31) ^ h3; } } @@ -76,13 +79,26 @@ public EventKind(LTTngContext context, uint id, string name, IReadOnlyList + /// Event ids are only unique within a stream (LTTng channel), so kinds are registered per stream. + /// + public static bool TryGetRegisteredKind(string domain, uint streamId, uint id, out EventKind kind) + { + return RegisteredKinds.TryGetValue(new Key(domain, streamId, id), out kind); } public static EventKind RegisterKind(LTTngContext context, uint id, string name, IReadOnlyList fields) + { + return RegisterKind(context, 0, id, name, fields); + } + + public static EventKind RegisterKind(LTTngContext context, uint streamId, uint id, string name, IReadOnlyList fields) { EventKind kind = new EventKind(context, id, name, fields); - RegisteredKinds.Add(new Key(context.Domain, id), kind); + RegisteredKinds.Add(new Key(context.Domain, streamId, id), kind); return kind; } } @@ -103,9 +119,9 @@ public LTTngGenericEvent(LTTngEvent data, LTTngContext context) this.CpuId = context.CurrentCpu; this.DiscardedEvents = data.DiscardedEvents; - if (!EventKind.TryGetRegisteredKind(context.Domain, data.Id, out this.kind)) + if (!EventKind.TryGetRegisteredKind(context.Domain, data.StreamId, data.Id, out this.kind)) { - this.kind = EventKind.RegisterKind(context, data.Id, data.Name, payload.Fields); + this.kind = EventKind.RegisterKind(context, data.StreamId, data.Id, data.Name, payload.Fields); } // As this is being written, all columns are of type 'T', so all rows are the same. For generic events, diff --git a/LinuxTraceLogCapture.md b/LinuxTraceLogCapture.md index 5371774..fdf5df0 100644 --- a/LinuxTraceLogCapture.md +++ b/LinuxTraceLogCapture.md @@ -44,7 +44,9 @@ Supports: - Block IO / Disk Activity - Diagnostic Messages -Once you have everything set up you just need to decide what kind of information you are looking for and begin tracing. +Traces in both the CTF 1.8 and CTF 2 formats are supported (CTF 2 is the default trace format since LTTng 2.15). + +Once you have everything set up you just need to decide what kind of information you are looking for and begin tracing. In this example we are looking at process scheduler events. We might use this to determine process lifetime and identify dependencies. You can learn more about what kind of "events" you can enable [here](https://lttng.org/man/1/lttng-enable-event/v2.8/). ```bash @@ -85,6 +87,12 @@ $ sudo lttng add-context --kernel --channel=channel0 --type=pid $ sudo lttng add-context --kernel --channel=channel0 --type=procname ``` +### Optionally, add CPU performance counters: +The Execution Events table shows how much each counter changed while a thread was running (Instruction Count, CPU Cycle, LLC Misses, Cache Misses). +```bash +$ sudo lttng add-context --kernel --channel=channel0 --type=perf:cpu:instructions --type=perf:cpu:cycles --type=perf:cpu:cache-misses --type=perf:cpu:LLC-load-misses +``` + ### Start the recording: ```bash $ sudo lttng start From 99222d3fdc637f14b77d1b4312b29469a210489a Mon Sep 17 00:00:00 2001 From: Hani Nemati Date: Thu, 8 Oct 2026 10:02:41 -0700 Subject: [PATCH 02/14] Honor big-endian byte order when reading floating point values CtfFloatingPointDescriptor passed the raw bytes to BitConverter regardless of its ByteOrder, so big-endian floats (now produced by the CTF 2 parser, and possible in CTF 1.8 via byte_order = be/network) decoded to wrong values. Reverse explicitly big-endian values before decoding, as CtfIntegerDescriptor does, and add tests. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Types/CtfFloatingPointDescriptor.cs | 15 ++++++ CtfUnitTest/Ctf2MetadataTests.cs | 18 +++++++- CtfUnitTest/FloatingPointTest.cs | 46 +++++++++++++++++++ 3 files changed, 78 insertions(+), 1 deletion(-) diff --git a/CtfPlayback/Metadata/Types/CtfFloatingPointDescriptor.cs b/CtfPlayback/Metadata/Types/CtfFloatingPointDescriptor.cs index 22a1f7e..f3a7d5b 100644 --- a/CtfPlayback/Metadata/Types/CtfFloatingPointDescriptor.cs +++ b/CtfPlayback/Metadata/Types/CtfFloatingPointDescriptor.cs @@ -31,6 +31,9 @@ internal CtfFloatingPointDescriptor(CtfPropertyBag bag) /// public string ByteOrder { get; } + private bool IsExplicitlyBigEndian => + StringComparer.Ordinal.Equals(this.ByteOrder, "be") || StringComparer.Ordinal.Equals(this.ByteOrder, "network"); + /// exp_dig is the number of digits represented in the exponent. public int Exponent { get; } @@ -72,6 +75,18 @@ internal CtfFieldValue Read(byte[] buffer, int byteCount) Debug.Assert(byteCount > 0); Debug.Assert(byteCount <= 8); // Up to 64-bits + // Values are decoded as little endian; reverse explicitly big-endian values first. + if (this.IsExplicitlyBigEndian) + { + var littleEndianBuffer = new byte[byteCount]; + for (int x = 0; x < byteCount; x++) + { + littleEndianBuffer[x] = buffer[byteCount - 1 - x]; + } + + buffer = littleEndianBuffer; + } + if (byteCount == 4) { var bufferAsInt = BitConverter.ToInt32(buffer); diff --git a/CtfUnitTest/Ctf2MetadataTests.cs b/CtfUnitTest/Ctf2MetadataTests.cs index 8e86ae3..f8d594d 100644 --- a/CtfUnitTest/Ctf2MetadataTests.cs +++ b/CtfUnitTest/Ctf2MetadataTests.cs @@ -35,7 +35,7 @@ public class Ctf2MetadataTests @"{""type"":""data-stream-class"",""id"":1,""default-clock-class-id"":""monotonic"",""packet-context-field-class"":""pkt-ctx"",""event-record-header-field-class"":""er-header-compact"",""event-record-common-context-field-class"":{""type"":""structure"",""member-classes"":[{""name"":""tid"",""field-class"":{""type"":""fixed-length-signed-integer"",""length"":32,""byte-order"":""little-endian"",""alignment"":8}}]}}", @"{""type"":""data-stream-class"",""id"":0,""default-clock-class-id"":""monotonic"",""packet-context-field-class"":""pkt-ctx"",""event-record-header-field-class"":""er-header-compact"",""event-record-common-context-field-class"":{""type"":""structure"",""member-classes"":[{""name"":""perf_cpu_cycles"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":64,""byte-order"":""little-endian"",""alignment"":8}},{""name"":""_callstack_kernel_length"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":32,""byte-order"":""little-endian"",""alignment"":8}},{""name"":""callstack_kernel"",""field-class"":{""type"":""dynamic-length-array"",""element-field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":64,""byte-order"":""little-endian"",""alignment"":8,""preferred-display-base"":16},""length-field-location"":{""path"":[""_callstack_kernel_length""]}}}]}}", @"{""type"":""event-record-class"",""data-stream-class-id"":1,""id"":0,""name"":""sched_wakeup"",""payload-field-class"":{""type"":""structure"",""member-classes"":[{""name"":""comm"",""field-class"":{""type"":""static-length-string"",""length"":16}},{""name"":""tid"",""field-class"":{""type"":""fixed-length-signed-integer"",""length"":32,""byte-order"":""little-endian"",""alignment"":8}}]}}", - @"{""type"":""event-record-class"",""data-stream-class-id"":0,""id"":0,""name"":""sched_switch"",""attributes"":{""lttng.org,2009"":{""log-level"":13}},""payload-field-class"":{""type"":""structure"",""member-classes"":[{""name"":""prev_tid"",""field-class"":{""type"":""fixed-length-signed-integer"",""length"":32,""byte-order"":""little-endian"",""alignment"":8}},{""name"":""filename"",""field-class"":{""type"":""null-terminated-string""}},{""name"":""address_ipv4"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":32,""byte-order"":""big-endian"",""alignment"":8,""preferred-display-base"":16}},{""name"":""ihl"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":4,""byte-order"":""big-endian""}},{""name"":""ratio"",""field-class"":{""type"":""fixed-length-floating-point-number"",""length"":64,""byte-order"":""little-endian"",""alignment"":8}}]}}", + @"{""type"":""event-record-class"",""data-stream-class-id"":0,""id"":0,""name"":""sched_switch"",""attributes"":{""lttng.org,2009"":{""log-level"":13}},""payload-field-class"":{""type"":""structure"",""member-classes"":[{""name"":""prev_tid"",""field-class"":{""type"":""fixed-length-signed-integer"",""length"":32,""byte-order"":""little-endian"",""alignment"":8}},{""name"":""filename"",""field-class"":{""type"":""null-terminated-string""}},{""name"":""address_ipv4"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":32,""byte-order"":""big-endian"",""alignment"":8,""preferred-display-base"":16}},{""name"":""ihl"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":4,""byte-order"":""big-endian""}},{""name"":""ratio"",""field-class"":{""type"":""fixed-length-floating-point-number"",""length"":64,""byte-order"":""little-endian"",""alignment"":8}},{""name"":""ratio_be"",""field-class"":{""type"":""fixed-length-floating-point-number"",""length"":32,""byte-order"":""big-endian"",""alignment"":8}}]}}", }; private static string LTTngKernelMetadata => string.Concat(LTTngKernelFragments.Select(fragment => "\u001e" + fragment + "\n")); @@ -177,6 +177,22 @@ public void ReadsBigEndianIntegers() Assert.AreEqual(0x7F000001ul, value.Value.ValueAsUlong); } + [TestMethod] + public void ReadsBigEndianFloatingPoint() + { + var builder = Parse(LTTngKernelMetadata); + var sched_switch = builder.AddedEvents.Single(e => e.Assignments["name"] == "sched_switch"); + var payload = (ICtfStructDescriptor)sched_switch.TypeDeclarations["fields"]; + + // 123.4f = 0x42F6CCCD + var bigEndian = (CtfFloatingPointDescriptor)payload.GetField("_ratio_be").TypeDescriptor; + Assert.AreEqual(123.4f, ((CtfFloatValue)bigEndian.Read(new byte[] { 0x42, 0xF6, 0xCC, 0xCD }, 4)).Value); + + // -2.0 = 0xC000000000000000 + var littleEndian = (CtfFloatingPointDescriptor)payload.GetField("_ratio").TypeDescriptor; + Assert.AreEqual(-2.0, ((CtfDoubleValue)littleEndian.Read(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0xC0 }, 8)).Value); + } + [TestMethod] public void ReadsVariableLengthIntegers() { diff --git a/CtfUnitTest/FloatingPointTest.cs b/CtfUnitTest/FloatingPointTest.cs index f7cce64..a7a8903 100644 --- a/CtfUnitTest/FloatingPointTest.cs +++ b/CtfUnitTest/FloatingPointTest.cs @@ -3,6 +3,8 @@ using System; using System.Collections.Generic; +using CtfPlayback.FieldValues; +using CtfPlayback.Metadata.InternalHelpers; using CtfPlayback.Metadata.Types; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -90,5 +92,49 @@ public void FloatingPointTests64() Assert.AreEqual(rawDoubleDict.Value, fpDouble); } } + + [TestMethod] + [TestCategory("UnitTest")] + [DataRow(null)] + [DataRow("le")] + [DataRow("native")] + [DataRow("be")] + [DataRow("network")] + public void FloatingPointByteOrder(string byteOrder) + { + bool bigEndian = byteOrder == "be" || byteOrder == "network"; + + // 123.4f = 0x42F6CCCD, -2.0 = 0xC000000000000000 + byte[] float32 = { 0xCD, 0xCC, 0xF6, 0x42 }; + byte[] float64 = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC0 }; + if (bigEndian) + { + Array.Reverse(float32); + Array.Reverse(float64); + } + + var single = (CtfFloatValue)CreateDescriptor(8, 24, byteOrder).Read(float32, float32.Length); + Assert.AreEqual(123.4f, single.Value); + + var dbl = (CtfDoubleValue)CreateDescriptor(11, 53, byteOrder).Read(float64, float64.Length); + Assert.AreEqual(-2.0, dbl.Value); + + // The caller's buffer must not be modified. + Assert.AreEqual(bigEndian ? (byte)0x42 : (byte)0xCD, float32[0]); + } + + private static CtfFloatingPointDescriptor CreateDescriptor(int exponent, int mantissa, string byteOrder) + { + var bag = new CtfPropertyBag(); + bag.AddValue("exp_dig", exponent.ToString()); + bag.AddValue("mant_dig", mantissa.ToString()); + bag.AddValue("align", "8"); + if (byteOrder != null) + { + bag.AddValue("byte_order", byteOrder); + } + + return new CtfFloatingPointDescriptor(bag); + } } } \ No newline at end of file From fd1227f34a414b767c96d0f87a06a2b18d1e24e7 Mon Sep 17 00:00:00 2001 From: Hani Nemati Date: Thu, 8 Oct 2026 10:19:22 -0700 Subject: [PATCH 03/14] Read packetized metadata in either byte order and with CTF 2 headers - Detect the packet magic number in little- or big-endian order and read content_size/packet_size in that order (explicit, not host byte order) - CTF 2 metadata packets (CTF2-PMETA-1.0) have a 44-byte header whose size is given by the header-size field; CTF 1.8 packets keep the 37-byte header - Decode the content of all packets at once so multi-byte UTF-8 characters split across packets are preserved - Pass the decoded text to the CTF 1.8 (ANTLR) parser so it no longer re-reads packets with its little-endian-only reader - Add tests for both byte orders and header versions, corrupt packets, and the repo's CTF 1.8 LTTng metadata re-packetized as big endian Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AntlrParser/CtfAntlrMetadataParser.cs | 10 ++ CtfPlayback/Metadata/CtfMetadataText.cs | 83 +++++++++++--- .../CtfVersionDetectingMetadataParser.cs | 7 +- CtfUnitTest/Ctf2MetadataTests.cs | 18 +-- CtfUnitTest/CtfMetadataTextTests.cs | 104 ++++++++++++++++++ CtfUnitTest/MetadataPackets.cs | 70 ++++++++++++ 6 files changed, 262 insertions(+), 30 deletions(-) create mode 100644 CtfUnitTest/CtfMetadataTextTests.cs create mode 100644 CtfUnitTest/MetadataPackets.cs diff --git a/CtfPlayback/Metadata/AntlrParser/CtfAntlrMetadataParser.cs b/CtfPlayback/Metadata/AntlrParser/CtfAntlrMetadataParser.cs index c3cc665..4b111f3 100644 --- a/CtfPlayback/Metadata/AntlrParser/CtfAntlrMetadataParser.cs +++ b/CtfPlayback/Metadata/AntlrParser/CtfAntlrMetadataParser.cs @@ -45,6 +45,16 @@ public ICtfMetadata Parse(Stream metadataStream) { string metadata = this.GetMetadata(metadataStream); + return this.ParseText(metadata); + } + + /// + /// Parse CTF metadata text that is not packetized. + /// + /// Metadata text + /// Metadata parsed from the text + internal ICtfMetadata ParseText(string metadata) + { this.parser = GetContext(metadata); // Error listeners can be switched out for debugging purposes. Leaving these comments here for easy access. diff --git a/CtfPlayback/Metadata/CtfMetadataText.cs b/CtfPlayback/Metadata/CtfMetadataText.cs index 435c52f..5c63891 100644 --- a/CtfPlayback/Metadata/CtfMetadataText.cs +++ b/CtfPlayback/Metadata/CtfMetadataText.cs @@ -2,20 +2,30 @@ // Licensed under the MIT License. using System; +using System.Buffers.Binary; using System.IO; using System.Text; namespace CtfPlayback.Metadata { /// - /// Helpers for reading a metadata stream, which may be plain text or packetized (see https://diamon.org/ctf/#spec7.1). + /// Helpers for reading a metadata stream, which may be plain text or packetized + /// (see https://diamon.org/ctf/#spec7.1 and https://diamon.org/ctf/files/CTF2-PMETA-1.0.html). /// internal static class CtfMetadataText { private const uint PacketMagic = 0x75d11d57; - // magic(4) + uuid(16) + checksum(4) + content_size(4) + packet_size(4) + 5 single-byte fields - private const int PacketHeaderSize = 37; + // CTF 1.8: magic(4) + uuid(16) + checksum(4) + content_size(4) + packet_size(4) + 5 single-byte fields + private const int Ctf1PacketHeaderSize = 37; + + // CTF2-PMETA-1.0: the CTF 1.8 fields, 3 reserved bytes, then the header size in bits (352 = 44 bytes). + private const int Ctf2MinimumPacketHeaderSize = 44; + + private const int ContentSizeOffset = 24; + private const int PacketSizeOffset = 28; + private const int MajorVersionOffset = 35; + private const int Ctf2HeaderSizeOffset = 40; internal static byte[] ReadAllBytes(Stream stream) { @@ -31,32 +41,79 @@ internal static byte[] ReadAllBytes(Stream stream) /// internal static string Read(byte[] metadata) { - if (metadata.Length < PacketHeaderSize || BitConverter.ToUInt32(metadata, 0) != PacketMagic) + if (metadata.Length < Ctf1PacketHeaderSize || !TryGetPacketByteOrder(metadata, 0, out _)) { return Encoding.UTF8.GetString(metadata); } - var sb = new StringBuilder(); - int offset = 0; - while (offset + PacketHeaderSize <= metadata.Length) + // Content is decoded once at the end because a multi-byte UTF-8 character may span two packets. + var content = new MemoryStream(metadata.Length); + long offset = 0; + while (offset + Ctf1PacketHeaderSize <= metadata.Length) { - if (BitConverter.ToUInt32(metadata, offset) != PacketMagic) + int packetOffset = (int)offset; + + // The encoding of the magic number gives the byte order of the header's other integer fields. + if (!TryGetPacketByteOrder(metadata, packetOffset, out bool bigEndian)) { throw new InvalidDataException("Metadata stream seems to be corrupt: invalid packet magic number."); } - int contentBytes = (int)(BitConverter.ToUInt32(metadata, offset + 24) / 8); - int packetBytes = (int)(BitConverter.ToUInt32(metadata, offset + 28) / 8); - if (contentBytes < PacketHeaderSize || packetBytes < contentBytes || offset + contentBytes > metadata.Length) + uint contentBits = ReadUInt32(metadata, packetOffset + ContentSizeOffset, bigEndian); + uint packetBits = ReadUInt32(metadata, packetOffset + PacketSizeOffset, bigEndian); + if ((contentBits % 8) != 0 || (packetBits % 8) != 0) + { + throw new InvalidDataException("Metadata stream seems to be corrupt: packet size is not a whole number of bytes."); + } + + int headerBytes = GetHeaderSize(metadata, packetOffset, bigEndian); + long contentBytes = contentBits / 8; + long packetBytes = packetBits / 8; + if (contentBytes < headerBytes || packetBytes < contentBytes || offset + contentBytes > metadata.Length) { throw new InvalidDataException("Metadata stream seems to be corrupt: invalid packet size."); } - sb.Append(Encoding.UTF8.GetString(metadata, offset + PacketHeaderSize, contentBytes - PacketHeaderSize)); + content.Write(metadata, packetOffset + headerBytes, (int)contentBytes - headerBytes); offset += packetBytes; } - return sb.ToString(); + return Encoding.UTF8.GetString(content.GetBuffer(), 0, (int)content.Length); + } + + private static int GetHeaderSize(byte[] metadata, int packetOffset, bool bigEndian) + { + // CTF 1.8 metadata packets are version 1.8; CTF 2 packets are version 2.0 and give their header size. + if (metadata[packetOffset + MajorVersionOffset] < 2) + { + return Ctf1PacketHeaderSize; + } + + if (packetOffset + Ctf2MinimumPacketHeaderSize > metadata.Length) + { + throw new InvalidDataException("Metadata stream seems to be corrupt: truncated packet header."); + } + + uint headerBits = ReadUInt32(metadata, packetOffset + Ctf2HeaderSizeOffset, bigEndian); + if ((headerBits % 8) != 0 || headerBits < Ctf2MinimumPacketHeaderSize * 8) + { + throw new InvalidDataException("Metadata stream seems to be corrupt: invalid packet header size."); + } + + return (int)(headerBits / 8); + } + + private static bool TryGetPacketByteOrder(byte[] metadata, int packetOffset, out bool bigEndian) + { + var magic = new ReadOnlySpan(metadata, packetOffset, sizeof(uint)); + bigEndian = BinaryPrimitives.ReadUInt32BigEndian(magic) == PacketMagic; + return bigEndian || BinaryPrimitives.ReadUInt32LittleEndian(magic) == PacketMagic; + } + + private static uint ReadUInt32(byte[] metadata, int index, bool bigEndian) + { + var bytes = new ReadOnlySpan(metadata, index, sizeof(uint)); + return bigEndian ? BinaryPrimitives.ReadUInt32BigEndian(bytes) : BinaryPrimitives.ReadUInt32LittleEndian(bytes); } } } diff --git a/CtfPlayback/Metadata/CtfVersionDetectingMetadataParser.cs b/CtfPlayback/Metadata/CtfVersionDetectingMetadataParser.cs index 82207df..811f6cc 100644 --- a/CtfPlayback/Metadata/CtfVersionDetectingMetadataParser.cs +++ b/CtfPlayback/Metadata/CtfVersionDetectingMetadataParser.cs @@ -46,6 +46,8 @@ public ICtfMetadata Parse(Stream metadataStream) } byte[] metadata = CtfMetadataText.ReadAllBytes(metadataStream); + + // Packetized metadata (in either byte order) is decoded here, for both CTF 1.8 and CTF 2. string text = CtfMetadataText.Read(metadata); if (Ctf2MetadataParser.IsCtf2Metadata(text)) @@ -53,10 +55,7 @@ public ICtfMetadata Parse(Stream metadataStream) return new Ctf2MetadataParser(this.metadataBuilder, this.prefixCtf2EventFieldNamesWithUnderscore).Parse(text); } - using (var memoryStream = new MemoryStream(metadata, false)) - { - return new CtfAntlrMetadataParser(this.metadataCustomization, this.metadataBuilder).Parse(memoryStream); - } + return new CtfAntlrMetadataParser(this.metadataCustomization, this.metadataBuilder).ParseText(text); } } } diff --git a/CtfUnitTest/Ctf2MetadataTests.cs b/CtfUnitTest/Ctf2MetadataTests.cs index f8d594d..24a051f 100644 --- a/CtfUnitTest/Ctf2MetadataTests.cs +++ b/CtfUnitTest/Ctf2MetadataTests.cs @@ -139,22 +139,14 @@ public void EventsAreIdentifiedByStreamAndId() } [TestMethod] - public void VersionDetectingParserHandlesPacketizedCtf2Metadata() + [DataRow(false)] + [DataRow(true)] + public void VersionDetectingParserHandlesPacketizedCtf2Metadata(bool bigEndian) { - byte[] text = Encoding.UTF8.GetBytes(LTTngKernelMetadata); - var stream = new MemoryStream(); - var writer = new BinaryWriter(stream); - writer.Write(0x75d11d57u); - writer.Write(new byte[16]); - writer.Write(0u); - writer.Write((uint)((37 + text.Length) * 8)); - writer.Write((uint)((37 + text.Length) * 8)); - writer.Write(new byte[] { 0, 0, 0, 2, 0 }); - writer.Write(text); - stream.Position = 0; + byte[] packets = MetadataPackets.Packetize(Encoding.UTF8.GetBytes(LTTngKernelMetadata), bigEndian, 2, maxContentBytesPerPacket: 512, paddingBytes: 4); var builder = new Ctf2TestMetadataBuilder(); - new CtfVersionDetectingMetadataParser(null, builder, true).Parse(stream); + new CtfVersionDetectingMetadataParser(null, builder, true).Parse(new MemoryStream(packets)); Assert.AreEqual(2, builder.Streams.Count); Assert.AreEqual(2, builder.AddedEvents.Count); diff --git a/CtfUnitTest/CtfMetadataTextTests.cs b/CtfUnitTest/CtfMetadataTextTests.cs new file mode 100644 index 0000000..c2c3234 --- /dev/null +++ b/CtfUnitTest/CtfMetadataTextTests.cs @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Buffers.Binary; +using System.IO; +using System.IO.Compression; +using System.Text; +using CtfPlayback.Metadata; +using CtfPlayback.Metadata.AntlrParser; +using CtfPlayback.Metadata.Interfaces; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace CtfUnitTest +{ + [TestClass] + public class CtfMetadataTextTests + { + // Contains multi-byte UTF-8 characters, which small packets split across packet boundaries. + private const string Text = + "\u001e{\"type\":\"preamble\",\"version\":2}\n" + + "\u001e{\"type\":\"trace-class\",\"environment\":{\"hostname\":\"h\u00f4te-\u00f1-\u65e5\u672c\"}}\n"; + + [TestMethod] + [DataRow(false, (byte)1, 1)] + [DataRow(true, (byte)1, 1)] + [DataRow(false, (byte)2, 1)] + [DataRow(true, (byte)2, 1)] + [DataRow(true, (byte)2, int.MaxValue)] + public void ReadsPacketizedMetadataInEitherByteOrder(bool bigEndian, byte majorVersion, int maxContentBytesPerPacket) + { + byte[] packets = MetadataPackets.Packetize(Encoding.UTF8.GetBytes(Text), bigEndian, majorVersion, maxContentBytesPerPacket, paddingBytes: 3); + + Assert.AreEqual(Text, CtfMetadataText.Read(packets)); + } + + [TestMethod] + public void ReturnsTextMetadataUnchanged() + { + Assert.AreEqual(Text, CtfMetadataText.Read(Encoding.UTF8.GetBytes(Text))); + + // Shorter than a packet header. + Assert.AreEqual("/* CTF 1.8 */", CtfMetadataText.Read(Encoding.UTF8.GetBytes("/* CTF 1.8 */"))); + } + + [TestMethod] + public void RejectsCorruptPackets() + { + byte[] content = Encoding.UTF8.GetBytes(Text); + + var badMagic = MetadataPackets.Packetize(content, true, 2, maxContentBytesPerPacket: 20); + badMagic[44 + 20] ^= 0xFF; + Assert.ThrowsException(() => CtfMetadataText.Read(badMagic)); + + var badHeaderSize = MetadataPackets.Packetize(content, true, 2); + BinaryPrimitives.WriteUInt32BigEndian(badHeaderSize.AsSpan(40), 37 * 8); + Assert.ThrowsException(() => CtfMetadataText.Read(badHeaderSize)); + + var badContentSize = MetadataPackets.Packetize(content, false, 1); + BinaryPrimitives.WriteUInt32LittleEndian(badContentSize.AsSpan(24), (uint)((37 + content.Length) * 8) - 3); + Assert.ThrowsException(() => CtfMetadataText.Read(badContentSize)); + + // A big-endian header read as little endian would give an absurd content size. + var bigEndian = MetadataPackets.Packetize(content, true, 1); + Assert.AreEqual(Text, CtfMetadataText.Read(bigEndian)); + } + + [TestMethod] + public void ParsesBigEndianPacketizedCtf18Metadata() + { + // Real LTTng CTF 1.8 metadata, stored as little-endian packets. + byte[] original; + using (var trace = ZipFile.OpenRead(@"..\..\..\..\TestData\LTTng\lttng-kernel-trace.ctf")) + using (var metadata = trace.GetEntry("kernel/metadata").Open()) + { + original = CtfMetadataText.ReadAllBytes(metadata); + } + + Assert.AreEqual(MetadataPackets.Magic, BinaryPrimitives.ReadUInt32LittleEndian(original)); + + string text = CtfMetadataText.Read(original); + var reference = new TestCtfMetadataCustomization(); + new CtfAntlrMetadataParser(reference, reference).Parse(new MemoryStream(original)); + + var littleEndian = ParseWithVersionDetectingParser(original); + var bigEndian = ParseWithVersionDetectingParser(MetadataPackets.Packetize(Encoding.UTF8.GetBytes(text), true, 1, 4096, 5)); + + Assert.IsTrue(reference.Events.Count > 0); + foreach (var parsed in new[] { littleEndian, bigEndian }) + { + Assert.AreEqual(reference.Events.Count, parsed.Events.Count); + Assert.AreEqual(reference.Streams.Count, parsed.Streams.Count); + Assert.AreEqual(reference.Clocks.Count, parsed.Clocks.Count); + Assert.AreEqual(reference.TraceDescriptor.Uuid, parsed.TraceDescriptor.Uuid); + } + } + + private static ICtfMetadata ParseWithVersionDetectingParser(byte[] metadata) + { + var builder = new TestCtfMetadataCustomization(); + return new CtfVersionDetectingMetadataParser(builder, builder, false).Parse(new MemoryStream(metadata)); + } + } +} diff --git a/CtfUnitTest/MetadataPackets.cs b/CtfUnitTest/MetadataPackets.cs new file mode 100644 index 0000000..37b11d1 --- /dev/null +++ b/CtfUnitTest/MetadataPackets.cs @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Buffers.Binary; +using System.IO; + +namespace CtfUnitTest +{ + /// + /// Builds packetized metadata streams (CTF 1.8 section 7.1, and CTF2-PMETA-1.0). + /// + internal static class MetadataPackets + { + internal const uint Magic = 0x75d11d57; + + /// Metadata content + /// Byte order of the header's integer fields + /// 1 for CTF 1.8 (37-byte header), 2 for CTF 2 (44-byte header) + /// Content is split into packets of at most this many bytes + /// Padding after the content of each packet + internal static byte[] Packetize(byte[] content, bool bigEndian, byte majorVersion, int maxContentBytesPerPacket = int.MaxValue, int paddingBytes = 0) + { + int headerSize = majorVersion >= 2 ? 44 : 37; + + var stream = new MemoryStream(); + int offset = 0; + do + { + int chunk = Math.Min(maxContentBytesPerPacket, content.Length - offset); + uint contentBits = (uint)((headerSize + chunk) * 8); + uint packetBits = (uint)((headerSize + chunk + paddingBytes) * 8); + + WriteUInt32(stream, Magic, bigEndian); + stream.Write(new byte[16], 0, 16); + WriteUInt32(stream, 0, bigEndian); + WriteUInt32(stream, contentBits, bigEndian); + WriteUInt32(stream, packetBits, bigEndian); + stream.Write(new byte[] { 0, 0, 0, majorVersion, majorVersion >= 2 ? (byte)0 : (byte)8 }, 0, 5); + if (majorVersion >= 2) + { + stream.Write(new byte[3], 0, 3); + WriteUInt32(stream, (uint)(headerSize * 8), bigEndian); + } + + stream.Write(content, offset, chunk); + stream.Write(new byte[paddingBytes], 0, paddingBytes); + offset += chunk; + } + while (offset < content.Length); + + return stream.ToArray(); + } + + internal static void WriteUInt32(Stream stream, uint value, bool bigEndian) + { + var bytes = new byte[4]; + if (bigEndian) + { + BinaryPrimitives.WriteUInt32BigEndian(bytes, value); + } + else + { + BinaryPrimitives.WriteUInt32LittleEndian(bytes, value); + } + + stream.Write(bytes, 0, bytes.Length); + } + } +} From bd6c7e08557b2af95a8b7823877e8394603c2631 Mon Sep 17 00:00:00 2001 From: Hani Nemati Date: Thu, 8 Oct 2026 10:28:52 -0700 Subject: [PATCH 04/14] Document the metadata packet magic number Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CtfPlayback/Metadata/CtfMetadataText.cs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CtfPlayback/Metadata/CtfMetadataText.cs b/CtfPlayback/Metadata/CtfMetadataText.cs index 5c63891..dab889b 100644 --- a/CtfPlayback/Metadata/CtfMetadataText.cs +++ b/CtfPlayback/Metadata/CtfMetadataText.cs @@ -14,7 +14,10 @@ namespace CtfPlayback.Metadata /// internal static class CtfMetadataText { - private const uint PacketMagic = 0x75d11d57; + // Magic number of a metadata packet, defined by CTF 1.8 section 7.1 (https://diamon.org/ctf/v1.8.3/#spec7.1) + // and CTF2-PMETA-1.0 (https://diamon.org/ctf/files/CTF2-PMETA-1.0.html). It may be stored in either byte order. + // This is not the magic number of data stream packets (0xC1FC1FC1). + private const uint MetadataPacketMagic = 0x75d11d57; // CTF 1.8: magic(4) + uuid(16) + checksum(4) + content_size(4) + packet_size(4) + 5 single-byte fields private const int Ctf1PacketHeaderSize = 37; @@ -106,8 +109,8 @@ private static int GetHeaderSize(byte[] metadata, int packetOffset, bool bigEndi private static bool TryGetPacketByteOrder(byte[] metadata, int packetOffset, out bool bigEndian) { var magic = new ReadOnlySpan(metadata, packetOffset, sizeof(uint)); - bigEndian = BinaryPrimitives.ReadUInt32BigEndian(magic) == PacketMagic; - return bigEndian || BinaryPrimitives.ReadUInt32LittleEndian(magic) == PacketMagic; + bigEndian = BinaryPrimitives.ReadUInt32BigEndian(magic) == MetadataPacketMagic; + return bigEndian || BinaryPrimitives.ReadUInt32LittleEndian(magic) == MetadataPacketMagic; } private static uint ReadUInt32(byte[] metadata, int index, bool bigEndian) From 93514218ab4ff1d2cb17cc8b7e40b436b6c3374e Mon Sep 17 00:00:00 2001 From: Hani Nemati Date: Thu, 8 Oct 2026 10:32:48 -0700 Subject: [PATCH 05/14] Fix CTF specification links https://diamon.org/ctf/ now hosts the CTF 2 specification, so the #spec7.1 and #spec4.1.7 anchors no longer exist there. Link CTF 1.8 sections to the v1.8.3 specification and CTF 2 metadata to its Metadata stream section. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CtfPlayback/Metadata/AntlrParser/CtfAntlrMetadataParser.cs | 2 +- CtfPlayback/Metadata/CtfMetadataText.cs | 3 ++- CtfUnitTest/FloatingPointTest.cs | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CtfPlayback/Metadata/AntlrParser/CtfAntlrMetadataParser.cs b/CtfPlayback/Metadata/AntlrParser/CtfAntlrMetadataParser.cs index 4b111f3..fdc7eeb 100644 --- a/CtfPlayback/Metadata/AntlrParser/CtfAntlrMetadataParser.cs +++ b/CtfPlayback/Metadata/AntlrParser/CtfAntlrMetadataParser.cs @@ -79,7 +79,7 @@ internal ICtfMetadata ParseText(string metadata) /// String representation of the metadata private unsafe string GetMetadata(Stream metadataStream) { - // See https://diamon.org/ctf/#spec7.1 + // See https://diamon.org/ctf/v1.8.3/#spec7.1 byte[] headerBuffer = new byte[Marshal.SizeOf(typeof(MetadataPacketHeader))]; byte[] buffer = null; diff --git a/CtfPlayback/Metadata/CtfMetadataText.cs b/CtfPlayback/Metadata/CtfMetadataText.cs index dab889b..a9473c1 100644 --- a/CtfPlayback/Metadata/CtfMetadataText.cs +++ b/CtfPlayback/Metadata/CtfMetadataText.cs @@ -10,7 +10,8 @@ namespace CtfPlayback.Metadata { /// /// Helpers for reading a metadata stream, which may be plain text or packetized - /// (see https://diamon.org/ctf/#spec7.1 and https://diamon.org/ctf/files/CTF2-PMETA-1.0.html). + /// (CTF 1.8: https://diamon.org/ctf/v1.8.3/#spec7.1, CTF 2: https://diamon.org/ctf/#metadata-stream and + /// https://diamon.org/ctf/files/CTF2-PMETA-1.0.html). /// internal static class CtfMetadataText { diff --git a/CtfUnitTest/FloatingPointTest.cs b/CtfUnitTest/FloatingPointTest.cs index a7a8903..169a419 100644 --- a/CtfUnitTest/FloatingPointTest.cs +++ b/CtfUnitTest/FloatingPointTest.cs @@ -28,7 +28,7 @@ public void FloatingPointTests32() { new byte[] { 0x00, 0x00, 0xC8, 0x3C }, 0.0244140625f}, // From https://en.wikipedia.org/wiki/Single-precision_floating-point_format - // CTF is supposed to follow IEEE 754-2008 format - https://diamon.org/ctf/#spec4.1.7 + // CTF is supposed to follow IEEE 754-2008 format - https://diamon.org/ctf/v1.8.3/#spec4.1.7 { new byte[] { 0x00, 0x00, 0x20, 0x3E }, 0.15625f }, { new byte[] { 0x00, 0x20, 0xA7, 0x44 }, 1337.0f }, { new byte[] { 0x00, 0x00, 0x46, 0x41 }, 12.375f }, From 3788e9f67f0305cb9aba79d9b30dc30e1fa4ac93 Mon Sep 17 00:00:00 2001 From: Hani Nemati Date: Thu, 8 Oct 2026 11:12:15 -0700 Subject: [PATCH 06/14] Move LEB128 decoding into a named helper Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Ctf2/Ctf2VariableLengthIntegerDescriptor.cs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/CtfPlayback/Metadata/Ctf2/Ctf2VariableLengthIntegerDescriptor.cs b/CtfPlayback/Metadata/Ctf2/Ctf2VariableLengthIntegerDescriptor.cs index 338770f..a9f9400 100644 --- a/CtfPlayback/Metadata/Ctf2/Ctf2VariableLengthIntegerDescriptor.cs +++ b/CtfPlayback/Metadata/Ctf2/Ctf2VariableLengthIntegerDescriptor.cs @@ -49,6 +49,16 @@ public override CtfFieldValue Read(IPacketReader reader, CtfFieldValue parent = reader.Align((uint)this.Align); + return new CtfIntegerValue(DecodeLeb128(reader, this.Signed), this); + } + + /// + /// Decodes an unsigned or signed LEB128 value (https://en.wikipedia.org/wiki/LEB128): each byte holds 7 bits + /// of the value, least significant group first, and its high bit is set when more bytes follow. A signed + /// value is sign-extended from bit 6 of its last byte. Bits beyond 64 are ignored. + /// + internal static IntegerLiteral DecodeLeb128(IPacketReader reader, bool signed) + { ulong value = 0; int shift = 0; byte current; @@ -64,9 +74,9 @@ public override CtfFieldValue Read(IPacketReader reader, CtfFieldValue parent = } while ((current & 0x80) != 0); - if (!this.Signed) + if (!signed) { - return new CtfIntegerValue(new IntegerLiteral(value), this); + return new IntegerLiteral(value); } if (shift < 64 && (current & 0x40) != 0) @@ -74,7 +84,7 @@ public override CtfFieldValue Read(IPacketReader reader, CtfFieldValue parent = value |= ulong.MaxValue << shift; } - return new CtfIntegerValue(new IntegerLiteral((long)value), this); + return new IntegerLiteral((long)value); } /// From 40faf3a01ad639ba4b6e8b1521ea6c9c4733c9e4 Mon Sep 17 00:00:00 2001 From: Hani Nemati Date: Thu, 8 Oct 2026 11:19:36 -0700 Subject: [PATCH 07/14] Note that array element parsing is shared by static and dynamic arrays Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CtfPlayback/Metadata/Ctf2/Ctf2MetadataParser.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CtfPlayback/Metadata/Ctf2/Ctf2MetadataParser.cs b/CtfPlayback/Metadata/Ctf2/Ctf2MetadataParser.cs index 306606e..43ec335 100644 --- a/CtfPlayback/Metadata/Ctf2/Ctf2MetadataParser.cs +++ b/CtfPlayback/Metadata/Ctf2/Ctf2MetadataParser.cs @@ -504,6 +504,8 @@ private static CtfMetadataTypeDescriptor BuildFloatingPoint(Dictionary fieldClass, BuildContext context) { var element = this.BuildFieldClass(GetRequiredValue(fieldClass, "element-field-class"), context); From a413613dc6fec25d6f56ff4fca6f3807268271fc Mon Sep 17 00:00:00 2001 From: Hani Nemati Date: Thu, 8 Oct 2026 11:43:59 -0700 Subject: [PATCH 08/14] Identify generic event kinds by event class instead of ids EventKind cached names and field names in a static dictionary keyed by (domain, stream id, event id). Ids are only unique within one stream of one trace, so traces of the same domain in one input (e.g. per-UID and per-PID UST buffers) and traces processed later in the same process reused the first trace's event names. The static Dictionary was also not thread safe. Cache kinds per event descriptor, which identifies the event class within its trace, in a ConditionalWeakTable so entries are released with the trace and lookups are thread safe. LTTngEvent now exposes its EventDescriptor. The legacy (domain, id) EventKind methods are kept for compatibility but marked obsolete; the (domain, stream, id) overloads added earlier in this change are removed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- LTTngCds/CookerData/LTTngEvent.cs | 6 + LTTngDataExtUnitTest/LTTngEventKindTests.cs | 143 ++++++++++++++++++ .../DataOutputTypes/LTTngGenericEvent.cs | 51 ++++--- 3 files changed, 178 insertions(+), 22 deletions(-) create mode 100644 LTTngDataExtUnitTest/LTTngEventKindTests.cs diff --git a/LTTngCds/CookerData/LTTngEvent.cs b/LTTngCds/CookerData/LTTngEvent.cs index e3de897..8ab23eb 100644 --- a/LTTngCds/CookerData/LTTngEvent.cs +++ b/LTTngCds/CookerData/LTTngEvent.cs @@ -59,6 +59,12 @@ internal LTTngEvent(LTTngEvent lttngEvent, long timestampAdjustmentNano) : this( /// public uint StreamId => this.ctfEvent.StreamId; + /// + /// Metadata describing the event's class. A single instance is shared by all events of that class within a + /// trace, so it identifies the event class even across traces whose stream and event ids overlap. + /// + public IEventDescriptor EventDescriptor => this.eventDescriptor; + /// /// Event name /// diff --git a/LTTngDataExtUnitTest/LTTngEventKindTests.cs b/LTTngDataExtUnitTest/LTTngEventKindTests.cs new file mode 100644 index 0000000..220b67c --- /dev/null +++ b/LTTngDataExtUnitTest/LTTngEventKindTests.cs @@ -0,0 +1,143 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text; +using LTTngDataExtensions.DataOutputTypes; +using LTTngDataExtensions.SourceDataCookers; +using Microsoft.Performance.SDK.Extensibility; +using Microsoft.Performance.SDK.Processing; +using Microsoft.Performance.Toolkit.Engine; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace LTTngDataExtUnitTest +{ + /// + /// Generic events must be named from their own trace's metadata, even when traces share a domain and their + /// stream and event ids overlap (e.g. per-UID and per-PID UST buffers), and when traces are processed one after + /// another in the same process. + /// + [TestClass] + public class LTTngEventKindTests + { + [TestMethod] + public void TracesInOneInputWithOverlappingEventIds() + { + var events = Process( + ("trace/ust/uid/1000/64-bit", "app_a_event", 1000, 1), + ("trace/ust/pid/app-42/64-bit", "app_b_event", 2000, 2)); + + CollectionAssert.AreEquivalent(new[] { "app_a_event = 1", "app_a_event = 1", "app_b_event = 2", "app_b_event = 2" }, events, string.Join(", ", events)); + } + + [TestMethod] + public void TracesProcessedOneAfterAnother() + { + var first = Process(("trace/ust/uid/1000/64-bit", "first_event", 1000, 1)); + var second = Process(("trace/ust/uid/1000/64-bit", "second_event", 1000, 2)); + + CollectionAssert.AreEqual(new[] { "first_event = 1", "first_event = 1" }, first, string.Join(", ", first)); + CollectionAssert.AreEqual(new[] { "second_event = 2", "second_event = 2" }, second, string.Join(", ", second)); + } + + private static List Process(params (string Directory, string EventName, uint Timestamp, int Value)[] traces) + { + string path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".ctf"); + try + { + using (var archive = ZipFile.Open(path, ZipArchiveMode.Create)) + { + foreach (var trace in traces) + { + WriteEntry(archive, trace.Directory + "/metadata", Encoding.UTF8.GetBytes(Metadata("ust", trace.EventName))); + WriteEntry(archive, trace.Directory + "/channel0_0", Packet(trace.Timestamp, trace.Value)); + } + } + + using (var runtime = Engine.Create(new FileDataSource(path))) + { + var cooker = new LTTngGenericEventDataCooker(); + runtime.EnableCooker(cooker.Path); + var results = runtime.Process(); + + Assert.AreEqual(0, results.ProcessingErrors.Count(), string.Join(Environment.NewLine, results.ProcessingErrors.Select(e => e.ProcessFault?.ToString()))); + + var events = results.QueryOutput>( + new DataOutputPath(cooker.Path, nameof(LTTngGenericEventDataCooker.Events))); + + return Enumerable.Range(0, (int)events.Count) + .Select(i => events[i]) + .Select(e => $"{e.EventName} = {e.FieldValues[e.FieldNames.IndexOf("_value")]}") + .ToList(); + } + } + finally + { + // The processor may still hold the archive open after the run. + try + { + File.Delete(path); + } + catch (IOException) + { + } + } + } + + // One data stream class (id 0) with one event record class (id 0) carrying a single integer. + private static string Metadata(string domain, string eventName) + { + return + "\u001e{\"type\":\"preamble\",\"version\":2}\n" + + "\u001e{\"type\":\"field-class-alias\",\"name\":\"u64-ts\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":64,\"alignment\":8,\"byte-order\":\"little-endian\",\"roles\":[\"default-clock-timestamp\"]}}\n" + + "\u001e{\"type\":\"field-class-alias\",\"name\":\"u64\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":64,\"alignment\":8,\"byte-order\":\"little-endian\"}}\n" + + "\u001e{\"type\":\"field-class-alias\",\"name\":\"u32\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":32,\"alignment\":8,\"byte-order\":\"little-endian\"}}\n" + + "\u001e{\"type\":\"field-class-alias\",\"name\":\"er-header-compact\",\"field-class\":{\"type\":\"structure\",\"minimum-alignment\":8,\"member-classes\":[{\"name\":\"id\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":5,\"byte-order\":\"little-endian\"}},{\"name\":\"v\",\"field-class\":{\"type\":\"variant\",\"selector-field-location\":{\"path\":[\"id\"]},\"options\":[{\"name\":\"compact\",\"selector-field-ranges\":[[0,30]],\"field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"timestamp\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":27,\"byte-order\":\"little-endian\",\"roles\":[\"default-clock-timestamp\"]}}]}},{\"name\":\"extended\",\"selector-field-ranges\":[[31,31]],\"field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"id\",\"field-class\":\"u32\"},{\"name\":\"timestamp\",\"field-class\":\"u64-ts\"}]}}]}}]}}\n" + + "\u001e{\"type\":\"trace-class\",\"environment\":{\"domain\":\"" + domain + "\",\"tracer_name\":\"lttng-ust\"},\"packet-header-field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"magic\",\"field-class\":\"u32\"},{\"name\":\"uuid\",\"field-class\":{\"type\":\"static-length-blob\",\"length\":16}},{\"name\":\"stream_id\",\"field-class\":\"u32\"}]}}\n" + + "\u001e{\"type\":\"clock-class\",\"id\":\"monotonic\",\"name\":\"monotonic\",\"frequency\":1000000000,\"origin\":\"unix-epoch\",\"offset-from-origin\":{\"seconds\":1790000000}}\n" + + "\u001e{\"type\":\"data-stream-class\",\"id\":0,\"default-clock-class-id\":\"monotonic\",\"event-record-header-field-class\":\"er-header-compact\",\"packet-context-field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"timestamp_begin\",\"field-class\":\"u64-ts\"},{\"name\":\"timestamp_end\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":64,\"alignment\":8,\"byte-order\":\"little-endian\",\"roles\":[\"packet-end-default-clock-timestamp\"]}},{\"name\":\"content_size\",\"field-class\":\"u64\"},{\"name\":\"packet_size\",\"field-class\":\"u64\"},{\"name\":\"events_discarded\",\"field-class\":\"u64\"},{\"name\":\"cpu_id\",\"field-class\":\"u32\"}]}}\n" + + "\u001e{\"type\":\"event-record-class\",\"data-stream-class-id\":0,\"id\":0,\"name\":\"" + eventName + "\",\"payload-field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"value\",\"field-class\":{\"type\":\"fixed-length-signed-integer\",\"length\":32,\"alignment\":8,\"byte-order\":\"little-endian\"}}]}}\n"; + } + + // A packet of stream 0 holding two events (id 0), at the given timestamp and 100 ns later. + private static byte[] Packet(uint timestamp, int value) + { + const int packetBytes = 4 + 16 + 4 + (5 * 8) + 4 + (2 * (4 + 4)); + ulong sizeInBits = packetBytes * 8; + + var stream = new MemoryStream(); + var writer = new BinaryWriter(stream); + writer.Write(0xC1FC1FC1u); + writer.Write(new byte[16]); + writer.Write(0u); + writer.Write((ulong)timestamp); + writer.Write((ulong)timestamp + 100); + writer.Write(sizeInBits); + writer.Write(sizeInBits); + writer.Write(0ul); + writer.Write(0u); + + // Compact event header: 5-bit event id (0) followed by a 27-bit timestamp. + foreach (uint eventTimestamp in new[] { timestamp, timestamp + 100 }) + { + writer.Write(eventTimestamp << 5); + writer.Write(value); + } + + Assert.AreEqual(packetBytes, stream.Length); + return stream.ToArray(); + } + + private static void WriteEntry(ZipArchive archive, string name, byte[] content) + { + using (var stream = archive.CreateEntry(name).Open()) + { + stream.Write(content, 0, content.Length); + } + } + } +} diff --git a/LTTngDataExtensions/DataOutputTypes/LTTngGenericEvent.cs b/LTTngDataExtensions/DataOutputTypes/LTTngGenericEvent.cs index 319414d..338754b 100644 --- a/LTTngDataExtensions/DataOutputTypes/LTTngGenericEvent.cs +++ b/LTTngDataExtensions/DataOutputTypes/LTTngGenericEvent.cs @@ -3,10 +3,12 @@ using System; using System.Collections.Generic; +using System.Runtime.CompilerServices; using System.Text.RegularExpressions; using CtfPlayback; using CtfPlayback.FieldValues; using LTTngCds.CookerData; +using LTTngCds.CtfExtensions.DescriptorInterfaces; using Microsoft.Performance.SDK; namespace LTTngDataExtensions.DataOutputTypes @@ -16,13 +18,11 @@ public class EventKind private class Key { private string domain; - private uint streamId; private uint id; - public Key(string domain, uint streamId, uint id) + public Key(string domain, uint id) { this.domain = domain ?? string.Empty; - this.streamId = streamId; this.id = id; } @@ -30,7 +30,7 @@ public override bool Equals(object obj) { if (obj is Key key) { - return domain.Equals(key.domain) && streamId.Equals(key.streamId) && id.Equals(key.id); + return domain.Equals(key.domain) && id.Equals(key.id); } else { @@ -42,8 +42,7 @@ public override int GetHashCode() { int h1 = domain.GetHashCode(); int h2 = id.GetHashCode(); - int h3 = streamId.GetHashCode(); - return ((((h1 << 5) + h1) ^ h2) * 31) ^ h3; + return ((h1 << 5) + h1) ^ h2; } } @@ -77,29 +76,40 @@ public EventKind(LTTngContext context, uint id, string name, IReadOnlyList RegisteredKinds = new Dictionary(); private static readonly Regex TraceLoggingEventRegex = new Regex("^(?[a-zA-Z_.0-9]+):(?[a-zA-Z_.0-9]+);(?.+);$"); - public static bool TryGetRegisteredKind(string domain, uint id, out EventKind kind) + // Event ids are only unique within one stream of one trace, and a single input can contain several traces of + // the same domain (e.g. per-UID and per-PID UST buffers). The event descriptor identifies the event class + // itself. Entries are released with the trace's metadata, so traces processed later or concurrently never + // share kinds. + private static readonly ConditionalWeakTable KindsByEventClass = new ConditionalWeakTable(); + + internal static EventKind GetKind(LTTngEvent data, LTTngContext context, IReadOnlyList fields) { - return TryGetRegisteredKind(domain, 0, id, out kind); + if (KindsByEventClass.TryGetValue(data.EventDescriptor, out var kind)) + { + return kind; + } + + return AddKind(data, context, fields); } - /// - /// Event ids are only unique within a stream (LTTng channel), so kinds are registered per stream. - /// - public static bool TryGetRegisteredKind(string domain, uint streamId, uint id, out EventKind kind) + [Obsolete("Event ids are not unique across streams or traces. LTTngGenericEvent identifies event kinds by event class instead.")] + public static bool TryGetRegisteredKind(string domain, uint id, out EventKind kind) { - return RegisteredKinds.TryGetValue(new Key(domain, streamId, id), out kind); + return RegisteredKinds.TryGetValue(new Key(domain, id), out kind); } + [Obsolete("Event ids are not unique across streams or traces. LTTngGenericEvent identifies event kinds by event class instead.")] public static EventKind RegisterKind(LTTngContext context, uint id, string name, IReadOnlyList fields) { - return RegisterKind(context, 0, id, name, fields); + EventKind kind = new EventKind(context, id, name, fields); + RegisteredKinds.Add(new Key(context.Domain, id), kind); + return kind; } - public static EventKind RegisterKind(LTTngContext context, uint streamId, uint id, string name, IReadOnlyList fields) + private static EventKind AddKind(LTTngEvent data, LTTngContext context, IReadOnlyList fields) { - EventKind kind = new EventKind(context, id, name, fields); - RegisteredKinds.Add(new Key(context.Domain, streamId, id), kind); - return kind; + // If another thread adds the same event class first, its kind is returned. + return KindsByEventClass.GetValue(data.EventDescriptor, _ => new EventKind(context, data.Id, data.Name, fields)); } } @@ -119,10 +129,7 @@ public LTTngGenericEvent(LTTngEvent data, LTTngContext context) this.CpuId = context.CurrentCpu; this.DiscardedEvents = data.DiscardedEvents; - if (!EventKind.TryGetRegisteredKind(context.Domain, data.StreamId, data.Id, out this.kind)) - { - this.kind = EventKind.RegisterKind(context, data.StreamId, data.Id, data.Name, payload.Fields); - } + this.kind = EventKind.GetKind(data, context, payload.Fields); // As this is being written, all columns are of type 'T', so all rows are the same. For generic events, // where columns have different types for different rows, this means everything becomes a string. From fcc447477cfd5cd4c58eaaa74abb7582aa0f9af0 Mon Sep 17 00:00:00 2001 From: Hani Nemati Date: Thu, 8 Oct 2026 11:55:57 -0700 Subject: [PATCH 09/14] Sign-extend signed integers wider than 32 bits correctly The sign mask was an int (1 << (Size - 1)), so for sizes above 32 bits the shift wrapped and the mask was sign-extended when widened to long. Signed integers of 33 to 64 bits, little- and big-endian, decoded to wrong values (e.g. 64-bit 0x0000000100000000 as -2147483648). Compute the sign bit in 64 bits and only extend values narrower than 64 bits. Decoding long.MinValue correctly then hit Math.Abs(long.MinValue) in IntegerLiteral's bit counting, so handle that value explicitly. Add tests for signed integers of every whole-byte size in both byte orders. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Metadata/Helpers/IntegerLiteral.cs | 6 ++ .../Metadata/Types/CtfIntegerDescriptor.cs | 14 ++- CtfUnitTest/IntegerDescriptorTests.cs | 97 +++++++++++++++++++ 3 files changed, 112 insertions(+), 5 deletions(-) create mode 100644 CtfUnitTest/IntegerDescriptorTests.cs diff --git a/CtfPlayback/Metadata/Helpers/IntegerLiteral.cs b/CtfPlayback/Metadata/Helpers/IntegerLiteral.cs index 7fbb77d..68587df 100644 --- a/CtfPlayback/Metadata/Helpers/IntegerLiteral.cs +++ b/CtfPlayback/Metadata/Helpers/IntegerLiteral.cs @@ -587,6 +587,12 @@ private static ushort HighestUsedBit(long value) ushort highestUsedBit = 0; + // Math.Abs cannot represent the magnitude of long.MinValue, which needs all 64 bits (63 plus the sign). + if (value == long.MinValue) + { + return 63; + } + // a negative number would always use the most significant bit value = Math.Abs(value); diff --git a/CtfPlayback/Metadata/Types/CtfIntegerDescriptor.cs b/CtfPlayback/Metadata/Types/CtfIntegerDescriptor.cs index ddbc514..2be1566 100644 --- a/CtfPlayback/Metadata/Types/CtfIntegerDescriptor.cs +++ b/CtfPlayback/Metadata/Types/CtfIntegerDescriptor.cs @@ -176,12 +176,16 @@ private IntegerLiteral ReadSignedLittleEndianValue(byte[] buffer, int byteCount) value |= buffer[x]; } - int signedMask = 1 << (this.Size - 1); - if ((value & signedMask) != 0) + // The sign bit must be computed in 64 bits: an int shift wraps for sizes above 32 bits. A 64-bit value + // already has its sign in place. + if (this.Size < 64) { - // extend the high order signed bit - long mask = ~(signedMask - 1); - value = value | mask; + long signBit = 1L << (this.Size - 1); + if ((value & signBit) != 0) + { + // extend the high order signed bit + value |= ~(signBit - 1); + } } return new IntegerLiteral(value); diff --git a/CtfUnitTest/IntegerDescriptorTests.cs b/CtfUnitTest/IntegerDescriptorTests.cs new file mode 100644 index 0000000..a5a50d9 --- /dev/null +++ b/CtfUnitTest/IntegerDescriptorTests.cs @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Linq; +using CtfPlayback.FieldValues; +using CtfPlayback.Metadata.InternalHelpers; +using CtfPlayback.Metadata.Types; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace CtfUnitTest +{ + [TestClass] + public class IntegerDescriptorTests + { + [TestMethod] + [DataRow("le")] + [DataRow("be")] + public void ReadsSignedIntegersOfEverySize(string byteOrder) + { + foreach (int size in new[] { 8, 16, 24, 32, 40, 48, 56, 64 }) + { + long max = size == 64 ? long.MaxValue : (1L << (size - 1)) - 1; + long min = size == 64 ? long.MinValue : -(1L << (size - 1)); + var values = new List { 0, 1, -1, max, min, max / 3, min / 3 }; + if (size > 32) + { + // Values with bits at and above bit 31 set, which a 32-bit sign mask mishandles. + values.AddRange(new[] { 1L << 31, 1L << 32, -(1L << 32), 0x7FFFFFFFL, 0x80000000L, -0x80000001L }); + } + + foreach (long value in values) + { + Assert.AreEqual(value, ReadSigned(size, byteOrder, value), $"size={size} byteOrder={byteOrder} value=0x{value:X}"); + } + } + } + + [TestMethod] + public void ReadsSigned64BitBigEndian() + { + var descriptor = CreateDescriptor(64, signed: true, "be"); + + var value = (CtfIntegerValue)descriptor.Read(new byte[] { 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00 }, 8); + Assert.AreEqual(0x0000000100000000L, value.Value.ValueAsLong); + + value = (CtfIntegerValue)descriptor.Read(new byte[] { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE }, 8); + Assert.AreEqual(-2L, value.Value.ValueAsLong); + + value = (CtfIntegerValue)descriptor.Read(new byte[] { 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 8); + Assert.IsTrue(value.Value.TryGetInt64(out long minimum)); + Assert.AreEqual(long.MinValue, minimum); + } + + [TestMethod] + [DataRow("le")] + [DataRow("be")] + public void ReadsUnsigned64Bit(string byteOrder) + { + foreach (ulong value in new[] { 0ul, 1ul, 0x80000000ul, 0x100000000ul, ulong.MaxValue }) + { + byte[] bytes = BitConverter.GetBytes(value); + if (byteOrder == "be") + { + Array.Reverse(bytes); + } + + var result = (CtfIntegerValue)CreateDescriptor(64, signed: false, byteOrder).Read(bytes, bytes.Length); + Assert.AreEqual(value, result.Value.ValueAsUlong); + } + } + + private static long ReadSigned(int size, string byteOrder, long value) + { + // Two's complement, truncated to the integer's size. + byte[] bytes = BitConverter.GetBytes(value).Take(size / 8).ToArray(); + if (byteOrder == "be") + { + Array.Reverse(bytes); + } + + var result = (CtfIntegerValue)CreateDescriptor(size, signed: true, byteOrder).Read(bytes, bytes.Length); + return result.Value.ValueAsLong; + } + + private static CtfIntegerDescriptor CreateDescriptor(int size, bool signed, string byteOrder) + { + var bag = new CtfPropertyBag(); + bag.AddValue("size", size.ToString()); + bag.AddValue("align", "8"); + bag.AddValue("signed", signed ? "true" : "false"); + bag.AddValue("byte_order", byteOrder); + return new CtfIntegerDescriptor(bag); + } + } +} From 2fd241174befea8a330d96be7791a6d3e35d947d Mon Sep 17 00:00:00 2001 From: Hani Nemati Date: Thu, 8 Oct 2026 12:13:57 -0700 Subject: [PATCH 10/14] Resolve CTF 2 field location origins instead of dropping them A field location may have an origin (e.g. event-record-payload), in which case its path starts at that scope's root structure rather than at the structure containing the field. The origin was ignored, so such paths were treated as relative and a length or selector inside a nested structure was not found when decoding. LTTng-UST metadata (written by lttng-tools) gives every field location an origin. Track the scope and the member path while building field classes. A location rooted at the same scope is converted to a path relative to the structure containing the field when that structure is on the path, which covers all locations LTTng writes. Locations in another scope, outside the containing structure, inside array elements or variant options, or with parent (null) steps are rejected with a clear error instead of being misread. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Metadata/Ctf2/Ctf2MetadataParser.cs | 102 +++++++++++----- CtfUnitTest/Ctf2MetadataTests.cs | 115 ++++++++++++++++++ 2 files changed, 188 insertions(+), 29 deletions(-) diff --git a/CtfPlayback/Metadata/Ctf2/Ctf2MetadataParser.cs b/CtfPlayback/Metadata/Ctf2/Ctf2MetadataParser.cs index 43ec335..1950313 100644 --- a/CtfPlayback/Metadata/Ctf2/Ctf2MetadataParser.cs +++ b/CtfPlayback/Metadata/Ctf2/Ctf2MetadataParser.cs @@ -181,7 +181,7 @@ private void AddTraceClass(Dictionary traceClass, Dictionary dataStreamClass) var eventHeaderClass = GetValue(dataStreamClass, "event-record-header-field-class") ?? throw new CtfMetadataException($"CTF 2 data stream class {id} does not define an event record header."); - var headerContext = BuildContext.Header(clockName); - - if (!(this.BuildFieldClass(packetContextClass, headerContext) is CtfStructDescriptor packetContext)) + if (!(this.BuildFieldClass(packetContextClass, BuildContext.Root("packet-context", clockName, false)) is CtfStructDescriptor packetContext)) { throw new CtfMetadataException($"CTF 2 data stream class {id} packet context is not a structure."); } - if (!(this.BuildFieldClass(eventHeaderClass, headerContext) is CtfStructDescriptor eventHeader)) + if (!(this.BuildFieldClass(eventHeaderClass, BuildContext.Root("event-record-header", clockName, false)) is CtfStructDescriptor eventHeader)) { throw new CtfMetadataException($"CTF 2 data stream class {id} event record header is not a structure."); } @@ -289,7 +287,8 @@ private void AddDataStreamClass(Dictionary dataStreamClass) var eventContextClass = GetValue(dataStreamClass, "event-record-common-context-field-class"); if (eventContextClass != null) { - eventContext = this.BuildFieldClass(eventContextClass, this.EventContext(clockName)) as CtfStructDescriptor + var commonContext = BuildContext.Root("event-record-common-context", clockName, this.prefixEventFieldNamesWithUnderscore); + eventContext = this.BuildFieldClass(eventContextClass, commonContext) as CtfStructDescriptor ?? throw new CtfMetadataException($"CTF 2 data stream class {id} event record common context is not a structure."); } @@ -319,18 +318,19 @@ private void AddEventRecordClass(Dictionary eventRecordClass) assignments["loglevel"] = logLevel; } - var context = this.EventContext(null); var typeDeclarations = new Dictionary(StringComparer.Ordinal); var payloadClass = GetValue(eventRecordClass, "payload-field-class"); typeDeclarations["fields"] = payloadClass != null - ? this.BuildFieldClass(payloadClass, context) + ? this.BuildFieldClass(payloadClass, BuildContext.Root("event-record-payload", null, this.prefixEventFieldNamesWithUnderscore)) : new CtfStructDescriptor(new CtfPropertyBag(), Array.Empty()); var specificContextClass = GetValue(eventRecordClass, "specific-context-field-class"); if (specificContextClass != null) { - typeDeclarations["context"] = this.BuildFieldClass(specificContextClass, context); + typeDeclarations["context"] = this.BuildFieldClass( + specificContextClass, + BuildContext.Root("event-record-specific-context", null, this.prefixEventFieldNamesWithUnderscore)); } this.metadataBuilder.AddEvent(assignments, typeDeclarations); @@ -341,11 +341,6 @@ private void AddEventRecordClass(Dictionary eventRecordClass) } } - private BuildContext EventContext(string clockName) - { - return new BuildContext(clockName, this.prefixEventFieldNamesWithUnderscore); - } - private CtfMetadataTypeDescriptor BuildFieldClass(object fieldClass, BuildContext context, IReadOnlyList selectorOptions = null) { var fieldClassObject = this.ResolveFieldClass(fieldClass); @@ -508,7 +503,7 @@ private static CtfMetadataTypeDescriptor BuildFloatingPoint(Dictionary fieldClass, BuildContext context) { - var element = this.BuildFieldClass(GetRequiredValue(fieldClass, "element-field-class"), context); + var element = this.BuildFieldClass(GetRequiredValue(fieldClass, "element-field-class"), context.NotAddressable()); if (TryGetNumber(fieldClass, "minimum-alignment", out var minimumAlignment) && minimumAlignment.AsLong() > element.Align) { throw new CtfMetadataException("CTF 2 arrays with a minimum alignment greater than their element alignment are not supported."); @@ -534,7 +529,8 @@ private CtfStructDescriptor BuildStructure(Dictionary fieldClass continue; } - string selectorName = GetSiblingSelectorName(memberClass); + string memberName = GetRequiredString(member, "name"); + string selectorName = GetSiblingSelectorName(memberClass, context.ForMember(memberName)); if (!selectorOptionsByMember.TryGetValue(selectorName, out var selectorOptions)) { selectorOptions = new List(); @@ -549,7 +545,7 @@ private CtfStructDescriptor BuildStructure(Dictionary fieldClass { string memberName = GetRequiredString(member, "name"); selectorOptionsByMember.TryGetValue(memberName, out var selectorOptions); - var memberType = this.BuildFieldClass(GetRequiredValue(member, "field-class"), context, selectorOptions); + var memberType = this.BuildFieldClass(GetRequiredValue(member, "field-class"), context.ForMember(memberName), selectorOptions); fields.Add(new CtfFieldDescriptor(memberType, context.MapName(memberName))); } @@ -560,7 +556,7 @@ private CtfStructDescriptor BuildStructure(Dictionary fieldClass private CtfVariantDescriptor BuildVariant(Dictionary fieldClass, BuildContext context) { - string selectorName = context.MapName(GetSiblingSelectorName(fieldClass)); + string selectorName = context.MapName(GetSiblingSelectorName(fieldClass, context)); var options = GetValue(fieldClass, "options") as List ?? throw new CtfMetadataException("CTF 2 variant does not define any options."); @@ -570,7 +566,7 @@ private CtfVariantDescriptor BuildVariant(Dictionary fieldClass, { var option = options[index] as Dictionary ?? throw new CtfMetadataException("CTF 2 variant option is not a JSON object."); - var optionType = this.BuildFieldClass(GetRequiredValue(option, "field-class"), context); + var optionType = this.BuildFieldClass(GetRequiredValue(option, "field-class"), context.NotAddressable()); fields.Add(new CtfFieldDescriptor(optionType, GetOptionName(option, index, context))); } @@ -610,9 +606,9 @@ private static string GetOptionName(Dictionary option, int index return name == null ? $"option{index}" : context.MapName(name); } - private static string GetSiblingSelectorName(Dictionary variantClass) + private static string GetSiblingSelectorName(Dictionary variantClass, BuildContext context) { - var path = GetFieldLocationPath(variantClass, "selector-field-location"); + var path = ResolveFieldLocation(variantClass, "selector-field-location", context); if (path.Count != 1) { throw new CtfMetadataException("Only CTF 2 variants whose selector is a sibling field are supported."); @@ -623,20 +619,48 @@ private static string GetSiblingSelectorName(Dictionary variantC private string GetFieldLocation(Dictionary fieldClass, string property, BuildContext context) { - return string.Join(".", GetFieldLocationPath(fieldClass, property).Select(context.MapName)); + return string.Join(".", ResolveFieldLocation(fieldClass, property, context).Select(context.MapName)); } - private static List GetFieldLocationPath(Dictionary fieldClass, string property) + /// + /// Returns the path of a field location (https://diamon.org/ctf/#field-loc) relative to the structure that + /// contains the field being built, which is how playback resolves it. + /// + private static List ResolveFieldLocation(Dictionary fieldClass, string property, BuildContext context) { if (!(GetValue(fieldClass, property) is Dictionary location) || - !(GetValue(location, "path") is List path) || - path.Count == 0) + !(GetValue(location, "path") is List pathElements) || + pathElements.Count == 0) { throw new CtfMetadataException($"CTF 2 field class has an invalid '{property}'."); } - return path.Select(element => element as string - ?? throw new CtfMetadataException($"CTF 2 '{property}' path elements must be field names.")).ToList(); + var path = pathElements.Select(element => element as string + ?? throw new CtfMetadataException($"CTF 2 '{property}' paths to a parent structure (null elements) are not supported.")).ToList(); + + string origin = GetString(location, "origin"); + if (origin == null) + { + // Without an origin, the path starts at the structure containing the field. + return path; + } + + // With an origin, the path starts at the root structure of that scope. LTTng-UST metadata writes all its + // locations this way, with the target in the same structure as the field. Such a path is made relative by + // removing the containing structure's own path. Targets in another scope, or outside the containing + // structure, would need lookups that playback doesn't support. + var container = context.Path?.Take(context.Path.Count - 1).ToList(); + if (!StringComparer.Ordinal.Equals(origin, context.Scope) || + container == null || + path.Count <= container.Count || + !path.Take(container.Count).SequenceEqual(container, StringComparer.Ordinal)) + { + throw new CtfMetadataException( + $"CTF 2 '{property}' with origin '{origin}' and path [{string.Join(", ", path)}] is not supported: " + + $"the target must be within the '{context.Scope}' structure that contains the field."); + } + + return path.Skip(container.Count).ToList(); } private static CtfIntegerDescriptor CreateByte(string encoding, int displayBase) @@ -763,17 +787,37 @@ private static Ctf2JsonNumber GetRequiredNumber(Dictionary json, private sealed class BuildContext { - internal BuildContext(string clockName, bool prefixNames) + private BuildContext(string scope, string clockName, bool prefixNames, IReadOnlyList path) { + this.Scope = scope; this.ClockName = clockName; this.PrefixNames = prefixNames; + this.Path = path; } + /// + /// The scope being built, named like a field location origin (e.g. "event-record-payload"). + /// + internal string Scope { get; } + internal string ClockName { get; } internal bool PrefixNames { get; } - internal static BuildContext Header(string clockName) => new BuildContext(clockName, false); + /// + /// Member names from the scope's root structure to the field class being built, or null when the field + /// can't be reached through structure members (array elements and variant options). + /// + internal IReadOnlyList Path { get; } + + internal static BuildContext Root(string scope, string clockName, bool prefixNames) => + new BuildContext(scope, clockName, prefixNames, Array.Empty()); + + internal BuildContext ForMember(string name) => + new BuildContext(this.Scope, this.ClockName, this.PrefixNames, this.Path?.Concat(new[] { name }).ToArray()); + + internal BuildContext NotAddressable() => + new BuildContext(this.Scope, this.ClockName, this.PrefixNames, null); internal string MapName(string name) => this.PrefixNames ? "_" + name : name; } diff --git a/CtfUnitTest/Ctf2MetadataTests.cs b/CtfUnitTest/Ctf2MetadataTests.cs index 24a051f..34ec32a 100644 --- a/CtfUnitTest/Ctf2MetadataTests.cs +++ b/CtfUnitTest/Ctf2MetadataTests.cs @@ -211,6 +211,89 @@ public void JsonReaderHandlesLargeNumbersAndEscapes() Assert.ThrowsException(() => Ctf2Json.Parse("{\"a\": }")); } + // Payload members written the way LTTng-UST metadata (lttng-tools) writes them: every field location has an + // origin, and its path starts at the root of that scope. + private const string OriginRootedPayload = + @"[{""name"":""len"",""field-class"":""u8""}," + + @"{""name"":""values"",""field-class"":{""type"":""dynamic-length-array"",""element-field-class"":""u32"",""length-field-location"":{""origin"":""event-record-payload"",""path"":[""len""]}}}," + + @"{""name"":""nested"",""field-class"":{""type"":""structure"",""member-classes"":[{""name"":""n"",""field-class"":""u8""},{""name"":""text"",""field-class"":{""type"":""dynamic-length-string"",""length-field-location"":{""origin"":""event-record-payload"",""path"":[""nested"",""n""]}}}]}}," + + @"{""name"":""sel"",""field-class"":""u8""}," + + @"{""name"":""choice"",""field-class"":{""type"":""variant"",""selector-field-location"":{""origin"":""event-record-payload"",""path"":[""sel""]},""options"":[{""name"":""a"",""selector-field-ranges"":[[0,0]],""field-class"":""u8""},{""name"":""b"",""selector-field-ranges"":[[1,1]],""field-class"":""u32""}]}}," + + @"{""name"":""after"",""field-class"":""u32""}]"; + + [TestMethod] + public void ResolvesFieldLocationsRootedAtTheSameScope() + { + var builder = Parse(MetadataWithPayload(OriginRootedPayload)); + + // The event record header variant uses an "event-record-header" origin. + var header = builder.Streams.Single().EventHeader; + Assert.AreEqual("id", ((ICtfVariantDescriptor)header.GetField("v").TypeDescriptor).Switch); + Assert.IsInstanceOfType(header.GetField("id").TypeDescriptor, typeof(ICtfEnumDescriptor)); + + var payload = (ICtfStructDescriptor)builder.AddedEvents.Single().TypeDeclarations["fields"]; + Assert.AreEqual("_len", ((ICtfArrayDescriptor)payload.GetField("_values").TypeDescriptor).Index); + + var nested = (ICtfStructDescriptor)payload.GetField("_nested").TypeDescriptor; + Assert.AreEqual("_n", ((ICtfArrayDescriptor)nested.GetField("_text").TypeDescriptor).Index); + + Assert.AreEqual("_sel", ((ICtfVariantDescriptor)payload.GetField("_choice").TypeDescriptor).Switch); + Assert.IsInstanceOfType(payload.GetField("_sel").TypeDescriptor, typeof(ICtfEnumDescriptor)); + } + + [TestMethod] + public void DecodesFieldsLocatedFromTheSameScope() + { + var payload = (ICtfStructDescriptor)Parse(MetadataWithPayload(OriginRootedPayload)).AddedEvents.Single().TypeDeclarations["fields"]; + + var bytes = new List { 2 }; + bytes.AddRange(BitConverter.GetBytes(10u)); + bytes.AddRange(BitConverter.GetBytes(20u)); + bytes.Add(3); + bytes.AddRange(Encoding.UTF8.GetBytes("abc")); + bytes.Add(1); + bytes.AddRange(BitConverter.GetBytes(0xDEADBEEFu)); + bytes.AddRange(BitConverter.GetBytes(7u)); + var reader = new BufferPacketReader(bytes.ToArray()); + + var value = (CtfStructValue)payload.Read(reader); + + CollectionAssert.AreEqual(new uint[] { 10, 20 }, value.ReadFieldAsArray("_values").ReadAsUInt32Array()); + Assert.AreEqual("abc", ((CtfStructValue)value.FieldsByName["_nested"]).ReadFieldAsArray("_text").ReadAsString()); + var choice = (CtfVariantValue)value.FieldsByName["_choice"]; + Assert.AreEqual("_b", choice.Identifier); + Assert.AreEqual(0xDEADBEEFu, ((CtfIntegerValue)choice.Value).Value.ValueAsUlong); + Assert.AreEqual(7u, value.ReadFieldAsUInt32("_after")); + Assert.IsTrue(reader.EndOfStream); + } + + [TestMethod] + [DataRow(@"[{""name"":""values"",""field-class"":{""type"":""dynamic-length-array"",""element-field-class"":""u32"",""length-field-location"":{""origin"":""event-record-common-context"",""path"":[""ctx_len""]}}}]", "event-record-common-context")] + [DataRow(@"[{""name"":""len"",""field-class"":""u8""},{""name"":""nested"",""field-class"":{""type"":""structure"",""member-classes"":[{""name"":""values"",""field-class"":{""type"":""dynamic-length-array"",""element-field-class"":""u32"",""length-field-location"":{""origin"":""event-record-payload"",""path"":[""len""]}}}]}}]", "event-record-payload")] + [DataRow(@"[{""name"":""items"",""field-class"":{""type"":""static-length-array"",""length"":2,""element-field-class"":{""type"":""structure"",""member-classes"":[{""name"":""n"",""field-class"":""u8""},{""name"":""values"",""field-class"":{""type"":""dynamic-length-array"",""element-field-class"":""u32"",""length-field-location"":{""origin"":""event-record-payload"",""path"":[""items"",""n""]}}}]}}}]", "event-record-payload")] + [DataRow(@"[{""name"":""len"",""field-class"":""u8""},{""name"":""nested"",""field-class"":{""type"":""structure"",""member-classes"":[{""name"":""values"",""field-class"":{""type"":""dynamic-length-array"",""element-field-class"":""u32"",""length-field-location"":{""path"":[null,""len""]}}}]}}]", "null elements")] + public void RejectsUnsupportedFieldLocations(string payloadMembers, string expectedMessage) + { + var exception = Assert.ThrowsException(() => Parse(MetadataWithPayload(payloadMembers))); + StringAssert.Contains(exception.Message, expectedMessage); + } + + private static string MetadataWithPayload(string payloadMembers) + { + return string.Concat(new[] + { + @"{""type"":""preamble"",""version"":2}", + @"{""type"":""field-class-alias"",""name"":""u8"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":8,""alignment"":8,""byte-order"":""little-endian""}}", + @"{""type"":""field-class-alias"",""name"":""u32"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":32,""alignment"":8,""byte-order"":""little-endian""}}", + @"{""type"":""trace-class"",""packet-header-field-class"":{""type"":""structure"",""member-classes"":[{""name"":""magic"",""field-class"":""u32""},{""name"":""stream_id"",""field-class"":""u32""}]}}", + @"{""type"":""data-stream-class"",""id"":0," + + @"""packet-context-field-class"":{""type"":""structure"",""member-classes"":[{""name"":""content_size"",""field-class"":""u32""}]}," + + @"""event-record-header-field-class"":{""type"":""structure"",""member-classes"":[{""name"":""id"",""field-class"":""u8""},{""name"":""v"",""field-class"":{""type"":""variant"",""selector-field-location"":{""origin"":""event-record-header"",""path"":[""id""]},""options"":[{""name"":""compact"",""selector-field-ranges"":[[0,254]],""field-class"":{""type"":""structure"",""member-classes"":[]}},{""name"":""extended"",""selector-field-ranges"":[[255,255]],""field-class"":{""type"":""structure"",""member-classes"":[{""name"":""id"",""field-class"":""u32""}]}}]}}]}," + + @"""event-record-common-context-field-class"":{""type"":""structure"",""member-classes"":[{""name"":""ctx_len"",""field-class"":""u32""}]}}", + @"{""type"":""event-record-class"",""data-stream-class-id"":0,""id"":0,""name"":""ust_event"",""payload-field-class"":{""type"":""structure"",""member-classes"":" + payloadMembers + "}}", + }.Select(fragment => "\u001e" + fragment + "\n")); + } + private static Ctf2TestMetadataBuilder Parse(string metadata) { var builder = new Ctf2TestMetadataBuilder(); @@ -259,6 +342,38 @@ public void AddEvent(IReadOnlyDictionary assignments, IReadOnlyD public void AddStream(ICtfStreamDescriptor streamDescriptor) => this.streams.Add(streamDescriptor); } + private sealed class BufferPacketReader + : IPacketReader + { + private readonly byte[] bytes; + private int position; + + public BufferPacketReader(byte[] bytes) + { + this.bytes = bytes; + } + + public bool EndOfStream => this.position >= this.bytes.Length; + + public uint RemainingBufferedBitCount => (uint)(this.bytes.Length - this.position) * 8; + + public byte[] ReadBits(uint bitCount) + { + Assert.AreEqual(0u, bitCount % 8, "Only whole bytes are read in these tests."); + var result = new byte[bitCount / 8]; + Array.Copy(this.bytes, this.position, result, 0, result.Length); + this.position += result.Length; + return result; + } + + public byte[] ReadString() => throw new NotSupportedException(); + + public void Align(uint bitCount) + { + Assert.AreEqual(0u, (uint)(this.position * 8) % bitCount, "These tests only use byte-aligned fields."); + } + } + private sealed class BytePacketReader : IPacketReader { From 54c99b797693fbbd573590e69375faab3e8859a8 Mon Sep 17 00:00:00 2001 From: Hani Nemati Date: Thu, 8 Oct 2026 12:24:21 -0700 Subject: [PATCH 11/14] Read LTTng's 37-byte CTF 2 metadata packet headers LTTng 2.15+ (lttng-modules and lttng-ust) packetizes CTF 2 metadata with the 37-byte CTF 1.8 header and sets the version to 2.0; it does not add the reserved bytes and header size field of CTF2-PMETA-1.0. Choosing the 44-byte layout from the major version alone read metadata text as the header size and rejected these traces. Use the CTF2-PMETA-1.0 header only when its header size field is valid for the packet. Metadata text can't contain NUL bytes and can't form a valid size for a real packet, so zero reserved bytes with an invalid size are still reported as corrupt; anything else uses the 37-byte header. The test packetizer now takes an explicit layout (CTF 1.8, LTTng CTF 2, CTF2-PMETA-1.0), and the packet tests cover LTTng's layout. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CtfPlayback/Metadata/CtfMetadataText.cs | 30 ++++++++++++++-------- CtfUnitTest/Ctf2MetadataTests.cs | 10 +++++--- CtfUnitTest/CtfMetadataTextTests.cs | 33 ++++++++++++++++--------- CtfUnitTest/MetadataPackets.cs | 29 +++++++++++++++++----- 4 files changed, 70 insertions(+), 32 deletions(-) diff --git a/CtfPlayback/Metadata/CtfMetadataText.cs b/CtfPlayback/Metadata/CtfMetadataText.cs index a9473c1..1500c38 100644 --- a/CtfPlayback/Metadata/CtfMetadataText.cs +++ b/CtfPlayback/Metadata/CtfMetadataText.cs @@ -20,7 +20,8 @@ internal static class CtfMetadataText // This is not the magic number of data stream packets (0xC1FC1FC1). private const uint MetadataPacketMagic = 0x75d11d57; - // CTF 1.8: magic(4) + uuid(16) + checksum(4) + content_size(4) + packet_size(4) + 5 single-byte fields + // CTF 1.8: magic(4) + uuid(16) + checksum(4) + content_size(4) + packet_size(4) + 5 single-byte fields. + // LTTng (2.15 and later) also uses this header for CTF 2 metadata, with version 2.0. private const int Ctf1PacketHeaderSize = 37; // CTF2-PMETA-1.0: the CTF 1.8 fields, 3 reserved bytes, then the header size in bits (352 = 44 bytes). @@ -29,6 +30,7 @@ internal static class CtfMetadataText private const int ContentSizeOffset = 24; private const int PacketSizeOffset = 28; private const int MajorVersionOffset = 35; + private const int Ctf2ReservedOffset = 37; private const int Ctf2HeaderSizeOffset = 40; internal static byte[] ReadAllBytes(Stream stream) @@ -70,9 +72,9 @@ internal static string Read(byte[] metadata) throw new InvalidDataException("Metadata stream seems to be corrupt: packet size is not a whole number of bytes."); } - int headerBytes = GetHeaderSize(metadata, packetOffset, bigEndian); long contentBytes = contentBits / 8; long packetBytes = packetBits / 8; + int headerBytes = GetHeaderSize(metadata, packetOffset, bigEndian, contentBytes); if (contentBytes < headerBytes || packetBytes < contentBytes || offset + contentBytes > metadata.Length) { throw new InvalidDataException("Metadata stream seems to be corrupt: invalid packet size."); @@ -85,26 +87,34 @@ internal static string Read(byte[] metadata) return Encoding.UTF8.GetString(content.GetBuffer(), 0, (int)content.Length); } - private static int GetHeaderSize(byte[] metadata, int packetOffset, bool bigEndian) + private static int GetHeaderSize(byte[] metadata, int packetOffset, bool bigEndian, long contentBytes) { - // CTF 1.8 metadata packets are version 1.8; CTF 2 packets are version 2.0 and give their header size. - if (metadata[packetOffset + MajorVersionOffset] < 2) + // Version 1.8 packets, and LTTng's version 2.0 packets, have the 37-byte CTF 1.8 header. Only version 2.0 + // packets with enough content can have the longer CTF2-PMETA-1.0 header. + if (metadata[packetOffset + MajorVersionOffset] < 2 || + contentBytes < Ctf2MinimumPacketHeaderSize || + packetOffset + Ctf2MinimumPacketHeaderSize > metadata.Length) { return Ctf1PacketHeaderSize; } - if (packetOffset + Ctf2MinimumPacketHeaderSize > metadata.Length) + uint headerBits = ReadUInt32(metadata, packetOffset + Ctf2HeaderSizeOffset, bigEndian); + if ((headerBits % 8) == 0 && headerBits >= Ctf2MinimumPacketHeaderSize * 8 && headerBits / 8 <= contentBytes) { - throw new InvalidDataException("Metadata stream seems to be corrupt: truncated packet header."); + return (int)(headerBits / 8); } - uint headerBits = ReadUInt32(metadata, packetOffset + Ctf2HeaderSizeOffset, bigEndian); - if ((headerBits % 8) != 0 || headerBits < Ctf2MinimumPacketHeaderSize * 8) + // In LTTng's layout these bytes are metadata text. Text never contains NUL bytes, so zero "reserved" bytes + // mean a CTF2-PMETA-1.0 header with a bad size. (Every text byte is at least 0x09, so text read as a header + // size is over 18 MB and can't pass the check above for a real metadata packet.) + if (metadata[packetOffset + Ctf2ReservedOffset] == 0 && + metadata[packetOffset + Ctf2ReservedOffset + 1] == 0 && + metadata[packetOffset + Ctf2ReservedOffset + 2] == 0) { throw new InvalidDataException("Metadata stream seems to be corrupt: invalid packet header size."); } - return (int)(headerBits / 8); + return Ctf1PacketHeaderSize; } private static bool TryGetPacketByteOrder(byte[] metadata, int packetOffset, out bool bigEndian) diff --git a/CtfUnitTest/Ctf2MetadataTests.cs b/CtfUnitTest/Ctf2MetadataTests.cs index 34ec32a..12be396 100644 --- a/CtfUnitTest/Ctf2MetadataTests.cs +++ b/CtfUnitTest/Ctf2MetadataTests.cs @@ -139,11 +139,13 @@ public void EventsAreIdentifiedByStreamAndId() } [TestMethod] - [DataRow(false)] - [DataRow(true)] - public void VersionDetectingParserHandlesPacketizedCtf2Metadata(bool bigEndian) + [DataRow(false, MetadataPacketLayout.LttngCtf2)] + [DataRow(true, MetadataPacketLayout.LttngCtf2)] + [DataRow(false, MetadataPacketLayout.Ctf2Pmeta)] + [DataRow(true, MetadataPacketLayout.Ctf2Pmeta)] + public void VersionDetectingParserHandlesPacketizedCtf2Metadata(bool bigEndian, MetadataPacketLayout layout) { - byte[] packets = MetadataPackets.Packetize(Encoding.UTF8.GetBytes(LTTngKernelMetadata), bigEndian, 2, maxContentBytesPerPacket: 512, paddingBytes: 4); + byte[] packets = MetadataPackets.Packetize(Encoding.UTF8.GetBytes(LTTngKernelMetadata), bigEndian, layout, maxContentBytesPerPacket: 512, paddingBytes: 4); var builder = new Ctf2TestMetadataBuilder(); new CtfVersionDetectingMetadataParser(null, builder, true).Parse(new MemoryStream(packets)); diff --git a/CtfUnitTest/CtfMetadataTextTests.cs b/CtfUnitTest/CtfMetadataTextTests.cs index c2c3234..947e4e3 100644 --- a/CtfUnitTest/CtfMetadataTextTests.cs +++ b/CtfUnitTest/CtfMetadataTextTests.cs @@ -22,14 +22,19 @@ public class CtfMetadataTextTests "\u001e{\"type\":\"trace-class\",\"environment\":{\"hostname\":\"h\u00f4te-\u00f1-\u65e5\u672c\"}}\n"; [TestMethod] - [DataRow(false, (byte)1, 1)] - [DataRow(true, (byte)1, 1)] - [DataRow(false, (byte)2, 1)] - [DataRow(true, (byte)2, 1)] - [DataRow(true, (byte)2, int.MaxValue)] - public void ReadsPacketizedMetadataInEitherByteOrder(bool bigEndian, byte majorVersion, int maxContentBytesPerPacket) + [DataRow(false, MetadataPacketLayout.Ctf18, 1)] + [DataRow(true, MetadataPacketLayout.Ctf18, 1)] + [DataRow(false, MetadataPacketLayout.LttngCtf2, 1)] + [DataRow(true, MetadataPacketLayout.LttngCtf2, 1)] + [DataRow(false, MetadataPacketLayout.LttngCtf2, 50)] + [DataRow(true, MetadataPacketLayout.LttngCtf2, int.MaxValue)] + [DataRow(false, MetadataPacketLayout.Ctf2Pmeta, 1)] + [DataRow(true, MetadataPacketLayout.Ctf2Pmeta, 1)] + [DataRow(false, MetadataPacketLayout.Ctf2Pmeta, 50)] + [DataRow(true, MetadataPacketLayout.Ctf2Pmeta, int.MaxValue)] + public void ReadsPacketizedMetadataInEitherByteOrder(bool bigEndian, MetadataPacketLayout layout, int maxContentBytesPerPacket) { - byte[] packets = MetadataPackets.Packetize(Encoding.UTF8.GetBytes(Text), bigEndian, majorVersion, maxContentBytesPerPacket, paddingBytes: 3); + byte[] packets = MetadataPackets.Packetize(Encoding.UTF8.GetBytes(Text), bigEndian, layout, maxContentBytesPerPacket, paddingBytes: 3); Assert.AreEqual(Text, CtfMetadataText.Read(packets)); } @@ -48,20 +53,24 @@ public void RejectsCorruptPackets() { byte[] content = Encoding.UTF8.GetBytes(Text); - var badMagic = MetadataPackets.Packetize(content, true, 2, maxContentBytesPerPacket: 20); + var badMagic = MetadataPackets.Packetize(content, true, MetadataPacketLayout.Ctf2Pmeta, maxContentBytesPerPacket: 20); badMagic[44 + 20] ^= 0xFF; Assert.ThrowsException(() => CtfMetadataText.Read(badMagic)); - var badHeaderSize = MetadataPackets.Packetize(content, true, 2); + var badHeaderSize = MetadataPackets.Packetize(content, true, MetadataPacketLayout.Ctf2Pmeta); BinaryPrimitives.WriteUInt32BigEndian(badHeaderSize.AsSpan(40), 37 * 8); Assert.ThrowsException(() => CtfMetadataText.Read(badHeaderSize)); - var badContentSize = MetadataPackets.Packetize(content, false, 1); + var headerLargerThanPacket = MetadataPackets.Packetize(content, false, MetadataPacketLayout.Ctf2Pmeta); + BinaryPrimitives.WriteUInt32LittleEndian(headerLargerThanPacket.AsSpan(40), 4096 * 8); + Assert.ThrowsException(() => CtfMetadataText.Read(headerLargerThanPacket)); + + var badContentSize = MetadataPackets.Packetize(content, false, MetadataPacketLayout.Ctf18); BinaryPrimitives.WriteUInt32LittleEndian(badContentSize.AsSpan(24), (uint)((37 + content.Length) * 8) - 3); Assert.ThrowsException(() => CtfMetadataText.Read(badContentSize)); // A big-endian header read as little endian would give an absurd content size. - var bigEndian = MetadataPackets.Packetize(content, true, 1); + var bigEndian = MetadataPackets.Packetize(content, true, MetadataPacketLayout.Ctf18); Assert.AreEqual(Text, CtfMetadataText.Read(bigEndian)); } @@ -83,7 +92,7 @@ public void ParsesBigEndianPacketizedCtf18Metadata() new CtfAntlrMetadataParser(reference, reference).Parse(new MemoryStream(original)); var littleEndian = ParseWithVersionDetectingParser(original); - var bigEndian = ParseWithVersionDetectingParser(MetadataPackets.Packetize(Encoding.UTF8.GetBytes(text), true, 1, 4096, 5)); + var bigEndian = ParseWithVersionDetectingParser(MetadataPackets.Packetize(Encoding.UTF8.GetBytes(text), true, MetadataPacketLayout.Ctf18, 4096, 5)); Assert.IsTrue(reference.Events.Count > 0); foreach (var parsed in new[] { littleEndian, bigEndian }) diff --git a/CtfUnitTest/MetadataPackets.cs b/CtfUnitTest/MetadataPackets.cs index 37b11d1..e6c9fe2 100644 --- a/CtfUnitTest/MetadataPackets.cs +++ b/CtfUnitTest/MetadataPackets.cs @@ -8,7 +8,22 @@ namespace CtfUnitTest { /// - /// Builds packetized metadata streams (CTF 1.8 section 7.1, and CTF2-PMETA-1.0). + /// Metadata packet header layouts. + /// + public enum MetadataPacketLayout + { + /// CTF 1.8 section 7.1: 37 bytes, version 1.8. + Ctf18, + + /// What LTTng 2.15+ writes for CTF 2 metadata: the 37-byte CTF 1.8 header with version 2.0. + LttngCtf2, + + /// CTF2-PMETA-1.0: 44 bytes (3 reserved bytes and a header size field added), version 2.0. + Ctf2Pmeta, + } + + /// + /// Builds packetized metadata streams. /// internal static class MetadataPackets { @@ -16,12 +31,14 @@ internal static class MetadataPackets /// Metadata content /// Byte order of the header's integer fields - /// 1 for CTF 1.8 (37-byte header), 2 for CTF 2 (44-byte header) + /// Packet header layout /// Content is split into packets of at most this many bytes /// Padding after the content of each packet - internal static byte[] Packetize(byte[] content, bool bigEndian, byte majorVersion, int maxContentBytesPerPacket = int.MaxValue, int paddingBytes = 0) + internal static byte[] Packetize(byte[] content, bool bigEndian, MetadataPacketLayout layout, int maxContentBytesPerPacket = int.MaxValue, int paddingBytes = 0) { - int headerSize = majorVersion >= 2 ? 44 : 37; + int headerSize = layout == MetadataPacketLayout.Ctf2Pmeta ? 44 : 37; + byte majorVersion = layout == MetadataPacketLayout.Ctf18 ? (byte)1 : (byte)2; + byte minorVersion = layout == MetadataPacketLayout.Ctf18 ? (byte)8 : (byte)0; var stream = new MemoryStream(); int offset = 0; @@ -36,8 +53,8 @@ internal static byte[] Packetize(byte[] content, bool bigEndian, byte majorVersi WriteUInt32(stream, 0, bigEndian); WriteUInt32(stream, contentBits, bigEndian); WriteUInt32(stream, packetBits, bigEndian); - stream.Write(new byte[] { 0, 0, 0, majorVersion, majorVersion >= 2 ? (byte)0 : (byte)8 }, 0, 5); - if (majorVersion >= 2) + stream.Write(new byte[] { 0, 0, 0, majorVersion, minorVersion }, 0, 5); + if (layout == MetadataPacketLayout.Ctf2Pmeta) { stream.Write(new byte[3], 0, 3); WriteUInt32(stream, (uint)(headerSize * 8), bigEndian); From eb4dd458ff00b348b9f5d2c7666f18cf0b67da60 Mon Sep 17 00:00:00 2001 From: Hani Nemati Date: Thu, 8 Oct 2026 13:02:08 -0700 Subject: [PATCH 12/14] Enhance validation for metadata packet sizes Added additional check for packetBytes to ensure it does not exceed metadata length. Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- CtfPlayback/Metadata/CtfMetadataText.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CtfPlayback/Metadata/CtfMetadataText.cs b/CtfPlayback/Metadata/CtfMetadataText.cs index 1500c38..d5a8770 100644 --- a/CtfPlayback/Metadata/CtfMetadataText.cs +++ b/CtfPlayback/Metadata/CtfMetadataText.cs @@ -75,7 +75,8 @@ internal static string Read(byte[] metadata) long contentBytes = contentBits / 8; long packetBytes = packetBits / 8; int headerBytes = GetHeaderSize(metadata, packetOffset, bigEndian, contentBytes); - if (contentBytes < headerBytes || packetBytes < contentBytes || offset + contentBytes > metadata.Length) + if (contentBytes < headerBytes || packetBytes < contentBytes || + offset + contentBytes > metadata.Length || offset + packetBytes > metadata.Length) { throw new InvalidDataException("Metadata stream seems to be corrupt: invalid packet size."); } @@ -84,6 +85,11 @@ internal static string Read(byte[] metadata) offset += packetBytes; } + if (offset != metadata.Length) + { + throw new InvalidDataException("Metadata stream seems to be corrupt: truncated packet header."); + } + return Encoding.UTF8.GetString(content.GetBuffer(), 0, (int)content.Length); } From 5127c5df056c94008548e31bccc938c96ed82d35 Mon Sep 17 00:00:00 2001 From: Hani Nemati Date: Thu, 8 Oct 2026 13:09:04 -0700 Subject: [PATCH 13/14] Expose and test CTF 2 event specific contexts The event specific context was read before the payload but nothing tested it, and LTTng never exposed its values. Add LTTngEvent.SpecificContext and include its fields in generic events, between the stream event context and the payload fields. Size the generic event field columns from all of an event's fields, not only its payload fields. Tests: - Parser: a specific context with mixed field sizes and a field location rooted at event-record-specific-context, decoded together with the payload from one buffer. - End to end: a CTF 2 trace with event record classes with and without a specific context in one stream; the context and following payload fields must decode correctly (a missed context read fails this test). - Share the synthetic CTF 2 trace writer between LTTng tests. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CtfUnitTest/Ctf2MetadataTests.cs | 39 ++++- LTTngCds/CookerData/LTTngEvent.cs | 6 + LTTngDataExtUnitTest/Ctf2TestTraces.cs | 160 ++++++++++++++++++ .../LTTngCtf2EventContextTests.cs | 63 +++++++ LTTngDataExtUnitTest/LTTngEventKindTests.cs | 115 ++----------- .../DataOutputTypes/LTTngGenericEvent.cs | 16 ++ .../LTTngGenericEventDataCooker.cs | 6 +- 7 files changed, 300 insertions(+), 105 deletions(-) create mode 100644 LTTngDataExtUnitTest/Ctf2TestTraces.cs create mode 100644 LTTngDataExtUnitTest/LTTngCtf2EventContextTests.cs diff --git a/CtfUnitTest/Ctf2MetadataTests.cs b/CtfUnitTest/Ctf2MetadataTests.cs index 12be396..14caa34 100644 --- a/CtfUnitTest/Ctf2MetadataTests.cs +++ b/CtfUnitTest/Ctf2MetadataTests.cs @@ -280,8 +280,43 @@ public void RejectsUnsupportedFieldLocations(string payloadMembers, string expec StringAssert.Contains(exception.Message, expectedMessage); } - private static string MetadataWithPayload(string payloadMembers) + [TestMethod] + public void ParsesAndDecodesEventSpecificContext() { + const string context = + @"[{""name"":""ctx_flag"",""field-class"":""u8""}," + + @"{""name"":""ctx_len"",""field-class"":""u8""}," + + @"{""name"":""ctx_values"",""field-class"":{""type"":""dynamic-length-array"",""element-field-class"":""u32"",""length-field-location"":{""origin"":""event-record-specific-context"",""path"":[""ctx_len""]}}}]"; + const string payload = @"[{""name"":""after"",""field-class"":""u32""}]"; + + var added = Parse(MetadataWithPayload(payload, context)).AddedEvents.Single(); + + var contextType = (ICtfStructDescriptor)added.TypeDeclarations["context"]; + CollectionAssert.AreEqual(new[] { "_ctx_flag", "_ctx_len", "_ctx_values" }, contextType.Fields.Select(f => f.Name).ToArray()); + Assert.AreEqual("_ctx_len", ((ICtfArrayDescriptor)contextType.GetField("_ctx_values").TypeDescriptor).Index); + + // An event's specific context is read right before its payload. + var bytes = new List { 9, 2 }; + bytes.AddRange(BitConverter.GetBytes(10u)); + bytes.AddRange(BitConverter.GetBytes(20u)); + bytes.AddRange(BitConverter.GetBytes(7u)); + var reader = new BufferPacketReader(bytes.ToArray()); + + var contextValue = (CtfStructValue)contextType.Read(reader); + var payloadValue = (CtfStructValue)((ICtfStructDescriptor)added.TypeDeclarations["fields"]).Read(reader); + + Assert.AreEqual(9u, contextValue.ReadFieldAsUInt32("_ctx_flag")); + CollectionAssert.AreEqual(new uint[] { 10, 20 }, contextValue.ReadFieldAsArray("_ctx_values").ReadAsUInt32Array()); + Assert.AreEqual(7u, payloadValue.ReadFieldAsUInt32("_after")); + Assert.IsTrue(reader.EndOfStream); + } + + private static string MetadataWithPayload(string payloadMembers, string specificContextMembers = null) + { + string specificContext = specificContextMembers == null + ? string.Empty + : @",""specific-context-field-class"":{""type"":""structure"",""member-classes"":" + specificContextMembers + "}"; + return string.Concat(new[] { @"{""type"":""preamble"",""version"":2}", @@ -292,7 +327,7 @@ private static string MetadataWithPayload(string payloadMembers) @"""packet-context-field-class"":{""type"":""structure"",""member-classes"":[{""name"":""content_size"",""field-class"":""u32""}]}," + @"""event-record-header-field-class"":{""type"":""structure"",""member-classes"":[{""name"":""id"",""field-class"":""u8""},{""name"":""v"",""field-class"":{""type"":""variant"",""selector-field-location"":{""origin"":""event-record-header"",""path"":[""id""]},""options"":[{""name"":""compact"",""selector-field-ranges"":[[0,254]],""field-class"":{""type"":""structure"",""member-classes"":[]}},{""name"":""extended"",""selector-field-ranges"":[[255,255]],""field-class"":{""type"":""structure"",""member-classes"":[{""name"":""id"",""field-class"":""u32""}]}}]}}]}," + @"""event-record-common-context-field-class"":{""type"":""structure"",""member-classes"":[{""name"":""ctx_len"",""field-class"":""u32""}]}}", - @"{""type"":""event-record-class"",""data-stream-class-id"":0,""id"":0,""name"":""ust_event"",""payload-field-class"":{""type"":""structure"",""member-classes"":" + payloadMembers + "}}", + @"{""type"":""event-record-class"",""data-stream-class-id"":0,""id"":0,""name"":""ust_event""" + specificContext + @",""payload-field-class"":{""type"":""structure"",""member-classes"":" + payloadMembers + "}}", }.Select(fragment => "\u001e" + fragment + "\n")); } diff --git a/LTTngCds/CookerData/LTTngEvent.cs b/LTTngCds/CookerData/LTTngEvent.cs index 8ab23eb..d253e84 100644 --- a/LTTngCds/CookerData/LTTngEvent.cs +++ b/LTTngCds/CookerData/LTTngEvent.cs @@ -95,6 +95,12 @@ internal LTTngEvent(LTTngEvent lttngEvent, long timestampAdjustmentNano) : this( /// public CtfStructValue StreamDefinedEventContext => this.ctfEvent.StreamDefinedEventContext as CtfStructValue; + /// + /// Context specific to the event's class (a CTF 2 event record specific context), read after the stream + /// defined event context and before the payload. Null when the event class doesn't define one. + /// + public CtfStructValue SpecificContext => this.ctfEvent.Context as CtfStructValue; + /// /// Event payload /// diff --git a/LTTngDataExtUnitTest/Ctf2TestTraces.cs b/LTTngDataExtUnitTest/Ctf2TestTraces.cs new file mode 100644 index 0000000..1e0150e --- /dev/null +++ b/LTTngDataExtUnitTest/Ctf2TestTraces.cs @@ -0,0 +1,160 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text; +using LTTngDataExtensions.DataOutputTypes; +using LTTngDataExtensions.SourceDataCookers; +using Microsoft.Performance.SDK.Extensibility; +using Microsoft.Performance.SDK.Processing; +using Microsoft.Performance.Toolkit.Engine; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace LTTngDataExtUnitTest +{ + /// + /// Builds small synthetic CTF 2 traces with one data stream class (id 0), and processes them. + /// + internal static class Ctf2TestTraces + { + internal const string StreamFileName = "channel0_0"; + + /// + /// Metadata with field class aliases u8, u16, u32, s32 and u64, and the given event record classes. + /// + internal static string Metadata(string domain, params string[] eventRecordClasses) + { + var fragments = new List + { + "{\"type\":\"preamble\",\"version\":2}", + "{\"type\":\"field-class-alias\",\"name\":\"u64-ts\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":64,\"alignment\":8,\"byte-order\":\"little-endian\",\"roles\":[\"default-clock-timestamp\"]}}", + Integer("u8", 8, false), + Integer("u16", 16, false), + Integer("u32", 32, false), + Integer("s32", 32, true), + Integer("u64", 64, false), + "{\"type\":\"field-class-alias\",\"name\":\"er-header-compact\",\"field-class\":{\"type\":\"structure\",\"minimum-alignment\":8,\"member-classes\":[{\"name\":\"id\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":5,\"byte-order\":\"little-endian\"}},{\"name\":\"v\",\"field-class\":{\"type\":\"variant\",\"selector-field-location\":{\"path\":[\"id\"]},\"options\":[{\"name\":\"compact\",\"selector-field-ranges\":[[0,30]],\"field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"timestamp\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":27,\"byte-order\":\"little-endian\",\"roles\":[\"default-clock-timestamp\"]}}]}},{\"name\":\"extended\",\"selector-field-ranges\":[[31,31]],\"field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"id\",\"field-class\":\"u32\"},{\"name\":\"timestamp\",\"field-class\":\"u64-ts\"}]}}]}}]}}", + "{\"type\":\"trace-class\",\"environment\":{\"domain\":\"" + domain + "\"},\"packet-header-field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"magic\",\"field-class\":\"u32\"},{\"name\":\"uuid\",\"field-class\":{\"type\":\"static-length-blob\",\"length\":16}},{\"name\":\"stream_id\",\"field-class\":\"u32\"}]}}", + "{\"type\":\"clock-class\",\"id\":\"monotonic\",\"name\":\"monotonic\",\"frequency\":1000000000,\"origin\":\"unix-epoch\",\"offset-from-origin\":{\"seconds\":1790000000}}", + "{\"type\":\"data-stream-class\",\"id\":0,\"default-clock-class-id\":\"monotonic\",\"event-record-header-field-class\":\"er-header-compact\",\"packet-context-field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"timestamp_begin\",\"field-class\":\"u64-ts\"},{\"name\":\"timestamp_end\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":64,\"alignment\":8,\"byte-order\":\"little-endian\",\"roles\":[\"packet-end-default-clock-timestamp\"]}},{\"name\":\"content_size\",\"field-class\":\"u64\"},{\"name\":\"packet_size\",\"field-class\":\"u64\"},{\"name\":\"events_discarded\",\"field-class\":\"u64\"},{\"name\":\"cpu_id\",\"field-class\":\"u32\"}]}}", + }; + + fragments.AddRange(eventRecordClasses); + return string.Concat(fragments.Select(fragment => "\u001e" + fragment + "\n")); + } + + /// + /// An event record class of data stream class 0. Members are JSON arrays of structure member classes. + /// + internal static string EventRecordClass(uint id, string name, string payloadMembers, string specificContextMembers = null) + { + string specificContext = specificContextMembers == null + ? string.Empty + : ",\"specific-context-field-class\":{\"type\":\"structure\",\"member-classes\":" + specificContextMembers + "}"; + + return "{\"type\":\"event-record-class\",\"data-stream-class-id\":0,\"id\":" + id + ",\"name\":\"" + name + "\"" + + specificContext + + ",\"payload-field-class\":{\"type\":\"structure\",\"member-classes\":" + payloadMembers + "}}"; + } + + /// + /// An event with a compact header (5-bit event record class id, then a 27-bit timestamp), followed by the + /// given context and payload bytes. + /// + internal static byte[] Event(uint id, uint timestamp, params byte[][] fields) + { + Assert.IsTrue(id < 31 && timestamp < (1u << 27)); + return BitConverter.GetBytes(id | (timestamp << 5)).Concat(fields.SelectMany(field => field)).ToArray(); + } + + /// + /// A packet of data stream 0 with the given events. + /// + internal static byte[] Packet(uint timestampBegin, uint timestampEnd, params byte[][] events) + { + const int headerAndContextBytes = 4 + 16 + 4 + (5 * 8) + 4; + ulong sizeInBits = (ulong)(headerAndContextBytes + events.Sum(e => e.Length)) * 8; + + var stream = new MemoryStream(); + var writer = new BinaryWriter(stream); + writer.Write(0xC1FC1FC1u); + writer.Write(new byte[16]); + writer.Write(0u); + writer.Write((ulong)timestampBegin); + writer.Write((ulong)timestampEnd); + writer.Write(sizeInBits); + writer.Write(sizeInBits); + writer.Write(0ul); + writer.Write(0u); + foreach (var e in events) + { + writer.Write(e); + } + + return stream.ToArray(); + } + + /// + /// Zips the traces (each a metadata file and one stream file in its directory), processes the archive, and + /// returns the generic events. + /// + internal static List ProcessGenericEvents(params (string Directory, string Metadata, byte[] Stream)[] traces) + { + string path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".ctf"); + try + { + using (var archive = ZipFile.Open(path, ZipArchiveMode.Create)) + { + foreach (var trace in traces) + { + WriteEntry(archive, trace.Directory + "/metadata", Encoding.UTF8.GetBytes(trace.Metadata)); + WriteEntry(archive, trace.Directory + "/" + StreamFileName, trace.Stream); + } + } + + using (var runtime = Engine.Create(new FileDataSource(path))) + { + var cooker = new LTTngGenericEventDataCooker(); + runtime.EnableCooker(cooker.Path); + var results = runtime.Process(); + + Assert.AreEqual(0, results.ProcessingErrors.Count(), string.Join(Environment.NewLine, results.ProcessingErrors.Select(e => e.ProcessFault?.ToString()))); + + var events = results.QueryOutput>( + new DataOutputPath(cooker.Path, nameof(LTTngGenericEventDataCooker.Events))); + + return Enumerable.Range(0, (int)events.Count).Select(i => events[i]).ToList(); + } + } + finally + { + // The processor may still hold the archive open after the run. + try + { + File.Delete(path); + } + catch (IOException) + { + } + } + } + + private static string Integer(string alias, int length, bool signed) + { + return "{\"type\":\"field-class-alias\",\"name\":\"" + alias + "\",\"field-class\":{\"type\":\"fixed-length-" + (signed ? "signed" : "unsigned") + + "-integer\",\"length\":" + length + ",\"alignment\":8,\"byte-order\":\"little-endian\"}}"; + } + + private static void WriteEntry(ZipArchive archive, string name, byte[] content) + { + using (var stream = archive.CreateEntry(name).Open()) + { + stream.Write(content, 0, content.Length); + } + } + } +} diff --git a/LTTngDataExtUnitTest/LTTngCtf2EventContextTests.cs b/LTTngDataExtUnitTest/LTTngCtf2EventContextTests.cs new file mode 100644 index 0000000..446385a --- /dev/null +++ b/LTTngDataExtUnitTest/LTTngCtf2EventContextTests.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Linq; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace LTTngDataExtUnitTest +{ + /// + /// A CTF 2 event record class can define a specific context, which is recorded between the stream's common + /// context and the payload. + /// + [TestClass] + public class LTTngCtf2EventContextTests + { + // Context members of different sizes, so a context that isn't read, or is read with the wrong size, shifts + // every payload field. + private const string ContextMembers = @"[{""name"":""ctx_flag"",""field-class"":""u8""},{""name"":""ctx_id"",""field-class"":""u32""}]"; + private const string PayloadMembers = @"[{""name"":""value"",""field-class"":""s32""},{""name"":""count"",""field-class"":""u16""}]"; + + [TestMethod] + public void DecodesSpecificContextAndFollowingPayload() + { + // Event record class 0 has a specific context; class 1 in the same stream doesn't. + string metadata = Ctf2TestTraces.Metadata( + "ust", + Ctf2TestTraces.EventRecordClass(0, "with_context", PayloadMembers, ContextMembers), + Ctf2TestTraces.EventRecordClass(1, "without_context", PayloadMembers)); + + byte[] stream = Ctf2TestTraces.Packet( + 1000, + 1200, + Ctf2TestTraces.Event(0, 1000, Context(7, 0x11223344), Payload(-5, 42)), + Ctf2TestTraces.Event(1, 1100, Payload(-6, 43)), + Ctf2TestTraces.Event(0, 1200, Context(8, 0x55667788), Payload(-7, 44))); + + var events = Ctf2TestTraces.ProcessGenericEvents(("trace/ust/uid/1000/64-bit", metadata, stream)) + .Select(e => $"{e.EventName}: " + string.Join(", ", e.FieldNames.Zip(e.FieldValues, (name, value) => $"{name} = {value}"))) + .ToList(); + + CollectionAssert.AreEqual( + new[] + { + "with_context: _ctx_flag = 7, _ctx_id = 287454020, _value = -5, _count = 42", + "without_context: _value = -6, _count = 43", + "with_context: _ctx_flag = 8, _ctx_id = 1432778632, _value = -7, _count = 44", + }, + events, + string.Join(Environment.NewLine, events)); + } + + private static byte[] Context(byte flag, uint id) + { + return new[] { flag }.Concat(BitConverter.GetBytes(id)).ToArray(); + } + + private static byte[] Payload(int value, ushort count) + { + return BitConverter.GetBytes(value).Concat(BitConverter.GetBytes(count)).ToArray(); + } + } +} diff --git a/LTTngDataExtUnitTest/LTTngEventKindTests.cs b/LTTngDataExtUnitTest/LTTngEventKindTests.cs index 220b67c..672d502 100644 --- a/LTTngDataExtUnitTest/LTTngEventKindTests.cs +++ b/LTTngDataExtUnitTest/LTTngEventKindTests.cs @@ -3,15 +3,7 @@ using System; using System.Collections.Generic; -using System.IO; -using System.IO.Compression; using System.Linq; -using System.Text; -using LTTngDataExtensions.DataOutputTypes; -using LTTngDataExtensions.SourceDataCookers; -using Microsoft.Performance.SDK.Extensibility; -using Microsoft.Performance.SDK.Processing; -using Microsoft.Performance.Toolkit.Engine; using Microsoft.VisualStudio.TestTools.UnitTesting; namespace LTTngDataExtUnitTest @@ -24,6 +16,8 @@ namespace LTTngDataExtUnitTest [TestClass] public class LTTngEventKindTests { + private const string ValuePayload = @"[{""name"":""value"",""field-class"":""s32""}]"; + [TestMethod] public void TracesInOneInputWithOverlappingEventIds() { @@ -44,100 +38,19 @@ public void TracesProcessedOneAfterAnother() CollectionAssert.AreEqual(new[] { "second_event = 2", "second_event = 2" }, second, string.Join(", ", second)); } + // Each trace has one event record class (id 0) and two events of it. private static List Process(params (string Directory, string EventName, uint Timestamp, int Value)[] traces) { - string path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".ctf"); - try - { - using (var archive = ZipFile.Open(path, ZipArchiveMode.Create)) - { - foreach (var trace in traces) - { - WriteEntry(archive, trace.Directory + "/metadata", Encoding.UTF8.GetBytes(Metadata("ust", trace.EventName))); - WriteEntry(archive, trace.Directory + "/channel0_0", Packet(trace.Timestamp, trace.Value)); - } - } - - using (var runtime = Engine.Create(new FileDataSource(path))) - { - var cooker = new LTTngGenericEventDataCooker(); - runtime.EnableCooker(cooker.Path); - var results = runtime.Process(); - - Assert.AreEqual(0, results.ProcessingErrors.Count(), string.Join(Environment.NewLine, results.ProcessingErrors.Select(e => e.ProcessFault?.ToString()))); - - var events = results.QueryOutput>( - new DataOutputPath(cooker.Path, nameof(LTTngGenericEventDataCooker.Events))); - - return Enumerable.Range(0, (int)events.Count) - .Select(i => events[i]) - .Select(e => $"{e.EventName} = {e.FieldValues[e.FieldNames.IndexOf("_value")]}") - .ToList(); - } - } - finally - { - // The processor may still hold the archive open after the run. - try - { - File.Delete(path); - } - catch (IOException) - { - } - } - } - - // One data stream class (id 0) with one event record class (id 0) carrying a single integer. - private static string Metadata(string domain, string eventName) - { - return - "\u001e{\"type\":\"preamble\",\"version\":2}\n" + - "\u001e{\"type\":\"field-class-alias\",\"name\":\"u64-ts\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":64,\"alignment\":8,\"byte-order\":\"little-endian\",\"roles\":[\"default-clock-timestamp\"]}}\n" + - "\u001e{\"type\":\"field-class-alias\",\"name\":\"u64\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":64,\"alignment\":8,\"byte-order\":\"little-endian\"}}\n" + - "\u001e{\"type\":\"field-class-alias\",\"name\":\"u32\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":32,\"alignment\":8,\"byte-order\":\"little-endian\"}}\n" + - "\u001e{\"type\":\"field-class-alias\",\"name\":\"er-header-compact\",\"field-class\":{\"type\":\"structure\",\"minimum-alignment\":8,\"member-classes\":[{\"name\":\"id\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":5,\"byte-order\":\"little-endian\"}},{\"name\":\"v\",\"field-class\":{\"type\":\"variant\",\"selector-field-location\":{\"path\":[\"id\"]},\"options\":[{\"name\":\"compact\",\"selector-field-ranges\":[[0,30]],\"field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"timestamp\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":27,\"byte-order\":\"little-endian\",\"roles\":[\"default-clock-timestamp\"]}}]}},{\"name\":\"extended\",\"selector-field-ranges\":[[31,31]],\"field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"id\",\"field-class\":\"u32\"},{\"name\":\"timestamp\",\"field-class\":\"u64-ts\"}]}}]}}]}}\n" + - "\u001e{\"type\":\"trace-class\",\"environment\":{\"domain\":\"" + domain + "\",\"tracer_name\":\"lttng-ust\"},\"packet-header-field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"magic\",\"field-class\":\"u32\"},{\"name\":\"uuid\",\"field-class\":{\"type\":\"static-length-blob\",\"length\":16}},{\"name\":\"stream_id\",\"field-class\":\"u32\"}]}}\n" + - "\u001e{\"type\":\"clock-class\",\"id\":\"monotonic\",\"name\":\"monotonic\",\"frequency\":1000000000,\"origin\":\"unix-epoch\",\"offset-from-origin\":{\"seconds\":1790000000}}\n" + - "\u001e{\"type\":\"data-stream-class\",\"id\":0,\"default-clock-class-id\":\"monotonic\",\"event-record-header-field-class\":\"er-header-compact\",\"packet-context-field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"timestamp_begin\",\"field-class\":\"u64-ts\"},{\"name\":\"timestamp_end\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":64,\"alignment\":8,\"byte-order\":\"little-endian\",\"roles\":[\"packet-end-default-clock-timestamp\"]}},{\"name\":\"content_size\",\"field-class\":\"u64\"},{\"name\":\"packet_size\",\"field-class\":\"u64\"},{\"name\":\"events_discarded\",\"field-class\":\"u64\"},{\"name\":\"cpu_id\",\"field-class\":\"u32\"}]}}\n" + - "\u001e{\"type\":\"event-record-class\",\"data-stream-class-id\":0,\"id\":0,\"name\":\"" + eventName + "\",\"payload-field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"value\",\"field-class\":{\"type\":\"fixed-length-signed-integer\",\"length\":32,\"alignment\":8,\"byte-order\":\"little-endian\"}}]}}\n"; - } - - // A packet of stream 0 holding two events (id 0), at the given timestamp and 100 ns later. - private static byte[] Packet(uint timestamp, int value) - { - const int packetBytes = 4 + 16 + 4 + (5 * 8) + 4 + (2 * (4 + 4)); - ulong sizeInBits = packetBytes * 8; - - var stream = new MemoryStream(); - var writer = new BinaryWriter(stream); - writer.Write(0xC1FC1FC1u); - writer.Write(new byte[16]); - writer.Write(0u); - writer.Write((ulong)timestamp); - writer.Write((ulong)timestamp + 100); - writer.Write(sizeInBits); - writer.Write(sizeInBits); - writer.Write(0ul); - writer.Write(0u); - - // Compact event header: 5-bit event id (0) followed by a 27-bit timestamp. - foreach (uint eventTimestamp in new[] { timestamp, timestamp + 100 }) - { - writer.Write(eventTimestamp << 5); - writer.Write(value); - } - - Assert.AreEqual(packetBytes, stream.Length); - return stream.ToArray(); - } - - private static void WriteEntry(ZipArchive archive, string name, byte[] content) - { - using (var stream = archive.CreateEntry(name).Open()) - { - stream.Write(content, 0, content.Length); - } + var events = Ctf2TestTraces.ProcessGenericEvents(traces.Select(trace => ( + trace.Directory, + Ctf2TestTraces.Metadata("ust", Ctf2TestTraces.EventRecordClass(0, trace.EventName, ValuePayload)), + Ctf2TestTraces.Packet( + trace.Timestamp, + trace.Timestamp + 100, + Ctf2TestTraces.Event(0, trace.Timestamp, BitConverter.GetBytes(trace.Value)), + Ctf2TestTraces.Event(0, trace.Timestamp + 100, BitConverter.GetBytes(trace.Value))))).ToArray()); + + return events.Select(e => $"{e.EventName} = {e.FieldValues[e.FieldNames.IndexOf("_value")]}").ToList(); } } -} +} \ No newline at end of file diff --git a/LTTngDataExtensions/DataOutputTypes/LTTngGenericEvent.cs b/LTTngDataExtensions/DataOutputTypes/LTTngGenericEvent.cs index 338754b..0a384c7 100644 --- a/LTTngDataExtensions/DataOutputTypes/LTTngGenericEvent.cs +++ b/LTTngDataExtensions/DataOutputTypes/LTTngGenericEvent.cs @@ -147,9 +147,15 @@ public LTTngGenericEvent(LTTngEvent data, LTTngContext context) fieldCount += data.StreamDefinedEventContext.Fields.Count; } + if (data.SpecificContext != null) + { + fieldCount += data.SpecificContext.Fields.Count; + } + this.FieldNames = new List(fieldCount); this.FieldValues = new List(fieldCount); + // Fields in the order they are recorded: stream context, event specific context, then payload. if (data.StreamDefinedEventContext != null) { foreach (var field in data.StreamDefinedEventContext.Fields) @@ -158,6 +164,16 @@ public LTTngGenericEvent(LTTngEvent data, LTTngContext context) this.FieldValues.Add(field.GetValueAsString()); } } + + if (data.SpecificContext != null) + { + foreach (var field in data.SpecificContext.Fields) + { + this.FieldNames.Add(field.FieldName.ToString()); + this.FieldValues.Add(field.GetValueAsString()); + } + } + foreach (var field in payload.Fields) { this.FieldNames.Add(field.FieldName.ToString()); diff --git a/LTTngDataExtensions/SourceDataCookers/LTTngGenericEventDataCooker.cs b/LTTngDataExtensions/SourceDataCookers/LTTngGenericEventDataCooker.cs index e4ae374..40b76c0 100644 --- a/LTTngDataExtensions/SourceDataCookers/LTTngGenericEventDataCooker.cs +++ b/LTTngDataExtensions/SourceDataCookers/LTTngGenericEventDataCooker.cs @@ -45,10 +45,12 @@ public override DataProcessingResult CookDataElement( { try { - Events.AddEvent(new LTTngGenericEvent(data, context)); + var genericEvent = new LTTngGenericEvent(data, context); + Events.AddEvent(genericEvent); + // Generic events list their context fields as well as their payload fields. this.MaximumEventFieldCount = - Math.Max(data.Payload.Fields.Count, this.MaximumEventFieldCount); + Math.Max(genericEvent.FieldNames.Count, this.MaximumEventFieldCount); } catch (CtfPlaybackException e) { From 7a302a945a19cdf086587c1256f955d1420fc23e Mon Sep 17 00:00:00 2001 From: Hani Nemati Date: Thu, 8 Oct 2026 13:25:38 -0700 Subject: [PATCH 14/14] Stop packing LTTngDriver as a plugin and fail on pack errors in CI LTTngDriver is a command-line tool that ships the Performance SDK and engine, so plugintool pack always fails for it (its build output also carries the LTTng plugin's manifest). The Create PTIX step only reported the exit code of its last command, and the hashtable's key order varies between runs, so the step failed only when LTTngDriver happened to be packed last. Don't pack LTTngDriver (its files are still published as an artifact), pack in a fixed order, and fail the step if any manifest is missing or any pack fails. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 75207b7..d0be7a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,11 +54,12 @@ jobs: - name: Create PTIX run: | - $dirs = @{ + # LTTngDriver is a command-line tool, not a WPA plugin: it ships the SDK and engine, so it can't be packed (its + # build output also carries the LTTng plugin's manifest). Its files are published by the next step. + $dirs = [ordered]@{ "PerfDataExtension" = "PerfDataExtension/bin/Release/netstandard2.1/" "PerfDataTxtExtension" = "PerfDataTxtExtension/bin/Release/netstandard2.1/" "LTTngDataExtensions" = "LTTngDataExtensions/bin/Release/netstandard2.1/" - "LTTngDriver" = "LTTngDriver/bin/Release/net8.0/" "Perfetto" = "PerfettoCds/bin/Release/netstandard2.1/" "Cloud-init" = "LinuxLogParsers/LinuxPlugins-MicrosoftPerformanceToolkSDK/Cloud-init/bin/Release/netstandard2.1/" "Dmesg" = "LinuxLogParsers/LinuxPlugins-MicrosoftPerformanceToolkSDK/DmesgIsoLog/bin/Release/netstandard2.1/" @@ -73,7 +74,7 @@ jobs: $pluginManifestPath = "$sourcePath/pluginManifest.json" if (!(Test-Path $pluginManifestPath)) { - Write-Error "pluginManifest.json not found in $sourcePath" + throw "pluginManifest.json not found in $sourcePath" } $pluginManifest = Get-Content -Path $pluginManifestPath | ConvertFrom-Json @@ -83,6 +84,11 @@ jobs: $packageName = "$pluginId-$pluginVersion.ptix" plugintool pack -s $sourcePath -o "$targetPath/$packageName" + + # Only the last command's exit code would otherwise decide the step's result. + if ($LASTEXITCODE -ne 0) { + throw "plugintool pack failed for $plugin (exit code $LASTEXITCODE)" + } } - name: Copy files and prepare artifacts