diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 75207b7..d0be7a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,11 +54,12 @@ jobs: - name: Create PTIX run: | - $dirs = @{ + # LTTngDriver is a command-line tool, not a WPA plugin: it ships the SDK and engine, so it can't be packed (its + # build output also carries the LTTng plugin's manifest). Its files are published by the next step. + $dirs = [ordered]@{ "PerfDataExtension" = "PerfDataExtension/bin/Release/netstandard2.1/" "PerfDataTxtExtension" = "PerfDataTxtExtension/bin/Release/netstandard2.1/" "LTTngDataExtensions" = "LTTngDataExtensions/bin/Release/netstandard2.1/" - "LTTngDriver" = "LTTngDriver/bin/Release/net8.0/" "Perfetto" = "PerfettoCds/bin/Release/netstandard2.1/" "Cloud-init" = "LinuxLogParsers/LinuxPlugins-MicrosoftPerformanceToolkSDK/Cloud-init/bin/Release/netstandard2.1/" "Dmesg" = "LinuxLogParsers/LinuxPlugins-MicrosoftPerformanceToolkSDK/DmesgIsoLog/bin/Release/netstandard2.1/" @@ -73,7 +74,7 @@ jobs: $pluginManifestPath = "$sourcePath/pluginManifest.json" if (!(Test-Path $pluginManifestPath)) { - Write-Error "pluginManifest.json not found in $sourcePath" + throw "pluginManifest.json not found in $sourcePath" } $pluginManifest = Get-Content -Path $pluginManifestPath | ConvertFrom-Json @@ -83,6 +84,11 @@ jobs: $packageName = "$pluginId-$pluginVersion.ptix" plugintool pack -s $sourcePath -o "$targetPath/$packageName" + + # Only the last command's exit code would otherwise decide the step's result. + if ($LASTEXITCODE -ne 0) { + throw "plugintool pack failed for $plugin (exit code $LASTEXITCODE)" + } } - name: Copy files and prepare artifacts diff --git a/CtfPlayback/EventStreams/CtfEvent.cs b/CtfPlayback/EventStreams/CtfEvent.cs index 73aa195..0be1fbb 100644 --- a/CtfPlayback/EventStreams/CtfEvent.cs +++ b/CtfPlayback/EventStreams/CtfEvent.cs @@ -5,6 +5,7 @@ using System.Diagnostics; using CtfPlayback.EventStreams.Interfaces; using CtfPlayback.FieldValues; +using CtfPlayback.Metadata; using CtfPlayback.Metadata.Interfaces; namespace CtfPlayback.EventStreams @@ -25,6 +26,8 @@ public CtfEvent(PacketReader packetReader, ICtfMetadata metadata, ICtfPacket own public ulong ByteOffsetWithinPacket { get; private set; } + public uint StreamId => this.owningPacket.StreamId; + public CtfTimestamp Timestamp { get; private set; } public CtfFieldValue StreamDefinedEventHeader { get; private set; } @@ -102,8 +105,7 @@ public void Read() private void ReadStreamEventHeader() { - uint streamIndex = this.owningPacket.StreamId; - var eventHeaderDescriptor = this.packetReader.Metadata.Streams[(int)streamIndex].EventHeader; + var eventHeaderDescriptor = this.packetReader.Metadata.GetStream(this.owningPacket.StreamId).EventHeader; Debug.Assert(eventHeaderDescriptor != null); StreamDefinedEventHeader = eventHeaderDescriptor.Read(this.packetReader); @@ -111,8 +113,7 @@ private void ReadStreamEventHeader() private void ReadStreamEventContext() { - uint streamIndex = this.owningPacket.StreamId; - var eventContextDescriptor = this.packetReader.Metadata.Streams[(int)streamIndex].EventContext; + var eventContextDescriptor = this.packetReader.Metadata.GetStream(this.owningPacket.StreamId).EventContext; if (eventContextDescriptor == null) { return; diff --git a/CtfPlayback/EventStreams/CtfPacket.cs b/CtfPlayback/EventStreams/CtfPacket.cs index a7c9d00..39a59f5 100644 --- a/CtfPlayback/EventStreams/CtfPacket.cs +++ b/CtfPlayback/EventStreams/CtfPacket.cs @@ -6,6 +6,7 @@ using System.IO; using CtfPlayback.EventStreams.Interfaces; using CtfPlayback.FieldValues; +using CtfPlayback.Metadata; using CtfPlayback.Metadata.Interfaces; namespace CtfPlayback.EventStreams @@ -61,10 +62,12 @@ public bool ReadPacketMetadata() this.PacketByteOffset = this.packetReader.CountOfBytesProcessed; ReadPacketHeader(); - ReadPacketContext(); + // The packet context layout depends on the stream, which is identified in the packet header. this.DetermineStreamIndex(); + ReadPacketContext(); + this.packetReader.SetPacketSize(this.packetReader.PlaybackCustomization.GetBitsInPacket(this)); this.BitsInPacketContent = @@ -116,7 +119,7 @@ private void ReadPacketHeader() private void ReadPacketContext() { - this.StreamPacketContext = this.Metadata.Streams[(int)this.StreamId].PacketContext.Read(this.packetReader) as CtfStructValue; + this.StreamPacketContext = this.Metadata.GetStream(this.StreamId).PacketContext.Read(this.packetReader) as CtfStructValue; } private ulong GetRemainingBits() @@ -135,6 +138,7 @@ private void DetermineStreamIndex() if (streamIdDescriptor == null) { this.StreamId = 0; + return; } bool streamIdFound = this.TracePacketHeader.FieldsByName.TryGetValue("stream_id", out var streamIdFieldValue); diff --git a/CtfPlayback/EventStreams/Interfaces/ICtfEvent.cs b/CtfPlayback/EventStreams/Interfaces/ICtfEvent.cs index c80a429..3775772 100644 --- a/CtfPlayback/EventStreams/Interfaces/ICtfEvent.cs +++ b/CtfPlayback/EventStreams/Interfaces/ICtfEvent.cs @@ -16,6 +16,11 @@ public interface ICtfEvent /// ulong ByteOffsetWithinPacket { get; } + /// + /// Id of the stream (stream class) that contains this event. + /// + uint StreamId { get; } + /// /// Timestamp associated with the event /// diff --git a/CtfPlayback/Metadata/AntlrParser/CtfAntlrMetadataParser.cs b/CtfPlayback/Metadata/AntlrParser/CtfAntlrMetadataParser.cs index c3cc665..fdc7eeb 100644 --- a/CtfPlayback/Metadata/AntlrParser/CtfAntlrMetadataParser.cs +++ b/CtfPlayback/Metadata/AntlrParser/CtfAntlrMetadataParser.cs @@ -45,6 +45,16 @@ public ICtfMetadata Parse(Stream metadataStream) { string metadata = this.GetMetadata(metadataStream); + return this.ParseText(metadata); + } + + /// + /// Parse CTF metadata text that is not packetized. + /// + /// Metadata text + /// Metadata parsed from the text + internal ICtfMetadata ParseText(string metadata) + { this.parser = GetContext(metadata); // Error listeners can be switched out for debugging purposes. Leaving these comments here for easy access. @@ -69,7 +79,7 @@ public ICtfMetadata Parse(Stream metadataStream) /// String representation of the metadata private unsafe string GetMetadata(Stream metadataStream) { - // See https://diamon.org/ctf/#spec7.1 + // See https://diamon.org/ctf/v1.8.3/#spec7.1 byte[] headerBuffer = new byte[Marshal.SizeOf(typeof(MetadataPacketHeader))]; byte[] buffer = null; diff --git a/CtfPlayback/Metadata/Ctf2/Ctf2Json.cs b/CtfPlayback/Metadata/Ctf2/Ctf2Json.cs new file mode 100644 index 0000000..9e0cf89 --- /dev/null +++ b/CtfPlayback/Metadata/Ctf2/Ctf2Json.cs @@ -0,0 +1,273 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text; + +namespace CtfPlayback.Metadata.Ctf2 +{ + /// + /// A JSON number, kept in its textual form so that 64-bit unsigned values are not truncated. + /// + internal sealed class Ctf2JsonNumber + { + internal Ctf2JsonNumber(string text) + { + this.Text = text; + } + + internal string Text { get; } + + internal bool IsNegative => this.Text.StartsWith("-", StringComparison.Ordinal); + + internal long AsLong() => long.Parse(this.Text, NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture); + + internal ulong AsULong() => ulong.Parse(this.Text, NumberStyles.None, CultureInfo.InvariantCulture); + + public override string ToString() => this.Text; + } + + /// + /// Minimal JSON reader used for CTF 2 metadata. Objects are returned as + /// of string to object, arrays as of object, + /// numbers as , and strings, booleans and null as their .NET equivalents. + /// + internal sealed class Ctf2Json + { + private readonly string text; + private int position; + + private Ctf2Json(string text) + { + this.text = text; + } + + internal static object Parse(string text) + { + var reader = new Ctf2Json(text); + reader.SkipWhitespace(); + var value = reader.ReadValue(); + reader.SkipWhitespace(); + if (reader.position != text.Length) + { + throw reader.Error("Unexpected trailing characters"); + } + + return value; + } + + private object ReadValue() + { + if (this.position >= this.text.Length) + { + throw this.Error("Unexpected end of JSON"); + } + + char c = this.text[this.position]; + switch (c) + { + case '{': + return this.ReadObject(); + case '[': + return this.ReadArray(); + case '"': + return this.ReadString(); + case 't': + this.Expect("true"); + return true; + case 'f': + this.Expect("false"); + return false; + case 'n': + this.Expect("null"); + return null; + default: + if (c == '-' || (c >= '0' && c <= '9')) + { + return this.ReadNumber(); + } + + throw this.Error($"Unexpected character '{c}'"); + } + } + + private Dictionary ReadObject() + { + var result = new Dictionary(StringComparer.Ordinal); + this.position++; + this.SkipWhitespace(); + if (this.TryConsume('}')) + { + return result; + } + + while (true) + { + this.SkipWhitespace(); + if (this.position >= this.text.Length || this.text[this.position] != '"') + { + throw this.Error("Expected property name"); + } + + string name = this.ReadString(); + this.SkipWhitespace(); + if (!this.TryConsume(':')) + { + throw this.Error("Expected ':'"); + } + + this.SkipWhitespace(); + result[name] = this.ReadValue(); + this.SkipWhitespace(); + + if (this.TryConsume(',')) + { + continue; + } + + if (this.TryConsume('}')) + { + return result; + } + + throw this.Error("Expected ',' or '}'"); + } + } + + private List ReadArray() + { + var result = new List(); + this.position++; + this.SkipWhitespace(); + if (this.TryConsume(']')) + { + return result; + } + + while (true) + { + this.SkipWhitespace(); + result.Add(this.ReadValue()); + this.SkipWhitespace(); + + if (this.TryConsume(',')) + { + continue; + } + + if (this.TryConsume(']')) + { + return result; + } + + throw this.Error("Expected ',' or ']'"); + } + } + + private string ReadString() + { + this.position++; + var sb = new StringBuilder(); + while (this.position < this.text.Length) + { + char c = this.text[this.position++]; + if (c == '"') + { + return sb.ToString(); + } + + if (c != '\\') + { + sb.Append(c); + continue; + } + + if (this.position >= this.text.Length) + { + break; + } + + char escaped = this.text[this.position++]; + switch (escaped) + { + case '"': sb.Append('"'); break; + case '\\': sb.Append('\\'); break; + case '/': sb.Append('/'); break; + case 'b': sb.Append('\b'); break; + case 'f': sb.Append('\f'); break; + case 'n': sb.Append('\n'); break; + case 'r': sb.Append('\r'); break; + case 't': sb.Append('\t'); break; + case 'u': + if (this.position + 4 > this.text.Length || + !int.TryParse(this.text.Substring(this.position, 4), NumberStyles.HexNumber, CultureInfo.InvariantCulture, out int codeUnit)) + { + throw this.Error("Invalid unicode escape"); + } + + sb.Append((char)codeUnit); + this.position += 4; + break; + default: + throw this.Error($"Invalid escape character '{escaped}'"); + } + } + + throw this.Error("Unterminated string"); + } + + private Ctf2JsonNumber ReadNumber() + { + int start = this.position; + while (this.position < this.text.Length) + { + char c = this.text[this.position]; + if ((c >= '0' && c <= '9') || c == '-' || c == '+' || c == '.' || c == 'e' || c == 'E') + { + this.position++; + continue; + } + + break; + } + + return new Ctf2JsonNumber(this.text.Substring(start, this.position - start)); + } + + private void Expect(string literal) + { + if (string.CompareOrdinal(this.text, this.position, literal, 0, literal.Length) != 0) + { + throw this.Error($"Expected '{literal}'"); + } + + this.position += literal.Length; + } + + private bool TryConsume(char c) + { + if (this.position < this.text.Length && this.text[this.position] == c) + { + this.position++; + return true; + } + + return false; + } + + private void SkipWhitespace() + { + while (this.position < this.text.Length && char.IsWhiteSpace(this.text[this.position])) + { + this.position++; + } + } + + private CtfMetadataException Error(string message) + { + return new CtfMetadataException($"Invalid CTF 2 metadata JSON: {message} at offset {this.position}."); + } + } +} diff --git a/CtfPlayback/Metadata/Ctf2/Ctf2MetadataParser.cs b/CtfPlayback/Metadata/Ctf2/Ctf2MetadataParser.cs new file mode 100644 index 0000000..1950313 --- /dev/null +++ b/CtfPlayback/Metadata/Ctf2/Ctf2MetadataParser.cs @@ -0,0 +1,838 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using CtfPlayback.Metadata.Helpers; +using CtfPlayback.Metadata.Interfaces; +using CtfPlayback.Metadata.InternalHelpers; +using CtfPlayback.Metadata.NamedScopes; +using CtfPlayback.Metadata.TypeInterfaces; +using CtfPlayback.Metadata.Types; + +namespace CtfPlayback.Metadata.Ctf2 +{ + /// + /// Parses CTF 2 metadata (a JSON text sequence of fragments, see https://diamon.org/ctf/) into the same + /// descriptor model used for CTF 1.8, so that the rest of the playback pipeline is format-agnostic. + /// + public sealed class Ctf2MetadataParser + : ICtfMetadataParser + { + /// + /// Each CTF 2 metadata fragment is preceded by this record separator (RFC 7464). + /// + public const char RecordSeparator = '\u001e'; + + private const string DefaultClockTimestampRole = "default-clock-timestamp"; + private const string PacketEndDefaultClockTimestampRole = "packet-end-default-clock-timestamp"; + + private readonly ICtfMetadataBuilder metadataBuilder; + private readonly bool prefixEventFieldNamesWithUnderscore; + + private readonly Dictionary aliases = new Dictionary(StringComparer.Ordinal); + + /// + /// Constructor. + /// + /// The object used to build metadata + /// + /// CTF 1.8 LTTng traces prefix event context and payload field names with '_'. Set this to produce + /// the same field names from CTF 2 metadata. + /// + public Ctf2MetadataParser(ICtfMetadataBuilder metadataBuilder, bool prefixEventFieldNamesWithUnderscore) + { + this.metadataBuilder = metadataBuilder ?? throw new ArgumentNullException(nameof(metadataBuilder)); + this.prefixEventFieldNamesWithUnderscore = prefixEventFieldNamesWithUnderscore; + } + + /// + /// Determines whether the metadata text is CTF 2 metadata. + /// + /// Metadata text, with any packetization removed + /// true if the text is a CTF 2 JSON text sequence + public static bool IsCtf2Metadata(string metadataText) + { + if (metadataText == null) + { + return false; + } + + foreach (char c in metadataText) + { + if (c == RecordSeparator) + { + return true; + } + + if (!char.IsWhiteSpace(c)) + { + return false; + } + } + + return false; + } + + /// + public ICtfMetadata Parse(Stream metadataStream) + { + if (metadataStream == null) + { + throw new ArgumentNullException(nameof(metadataStream)); + } + + string text = CtfMetadataText.Read(CtfMetadataText.ReadAllBytes(metadataStream)); + return this.Parse(text); + } + + /// + /// Parses CTF 2 metadata text. + /// + /// Metadata text, with any packetization removed + /// The parsed metadata + public ICtfMetadata Parse(string metadataText) + { + var fragments = metadataText + .Split(RecordSeparator) + .Where(fragment => !string.IsNullOrWhiteSpace(fragment)) + .Select(fragment => Ctf2Json.Parse(fragment) as Dictionary + ?? throw new CtfMetadataException("CTF 2 metadata fragment is not a JSON object.")) + .ToList(); + + if (fragments.Count == 0 || GetString(fragments[0], "type") != "preamble") + { + throw new CtfMetadataException("CTF 2 metadata does not start with a preamble fragment."); + } + + var preamble = fragments[0]; + if (!TryGetNumber(preamble, "version", out var version) || version.AsLong() != 2) + { + throw new CtfMetadataException($"Unsupported CTF metadata version: {GetValue(preamble, "version")}."); + } + + Dictionary traceClass = null; + var clockClasses = new List>(); + var dataStreamClasses = new List>(); + var eventRecordClasses = new List>(); + + foreach (var fragment in fragments.Skip(1)) + { + string type = GetString(fragment, "type"); + switch (type) + { + case "field-class-alias": + this.aliases[GetRequiredString(fragment, "name")] = GetRequiredValue(fragment, "field-class"); + break; + case "trace-class": + traceClass = fragment; + break; + case "clock-class": + clockClasses.Add(fragment); + break; + case "data-stream-class": + dataStreamClasses.Add(fragment); + break; + case "event-record-class": + eventRecordClasses.Add(fragment); + break; + case "preamble": + throw new CtfMetadataException("CTF 2 metadata contains more than one preamble fragment."); + default: + // Unknown fragment types are ignored so newer metadata can still be read. + break; + } + } + + if (traceClass == null) + { + throw new CtfMetadataException("CTF 2 metadata does not contain a trace-class fragment."); + } + + this.AddTraceClass(traceClass, preamble); + + foreach (var clockClass in clockClasses) + { + this.metadataBuilder.AddClock(CreateClock(clockClass)); + } + + // Streams are looked up by id, and CTF 1.8 streams are declared in id order. Keep that ordering. + foreach (var dataStreamClass in dataStreamClasses.OrderBy(GetDataStreamClassId)) + { + this.AddDataStreamClass(dataStreamClass); + } + + foreach (var eventRecordClass in eventRecordClasses) + { + this.AddEventRecordClass(eventRecordClass); + } + + return this.metadataBuilder; + } + + private void AddTraceClass(Dictionary traceClass, Dictionary preamble) + { + var packetHeaderClass = GetValue(traceClass, "packet-header-field-class"); + if (packetHeaderClass == null) + { + throw new CtfMetadataException("CTF 2 trace-class does not define a packet header field class."); + } + + if (!(this.BuildFieldClass(packetHeaderClass, BuildContext.Root("packet-header", null, false)) is CtfStructDescriptor packetHeader)) + { + throw new CtfMetadataException("CTF 2 trace-class packet header field class is not a structure."); + } + + var traceBag = new CtfPropertyBag(); + traceBag.AddValue("major", "2"); + traceBag.AddValue("minor", "0"); + traceBag.AddValue("byte_order", "le"); + string uid = GetString(traceClass, "uid") ?? FormatUuid(GetValue(preamble, "uuid")); + if (uid != null && Guid.TryParse(uid, out _)) + { + traceBag.AddValue("uuid", uid); + } + + this.metadataBuilder.SetTraceDescriptor(new CtfTraceDescriptor(traceBag, packetHeader)); + + var environmentBag = new CtfPropertyBag(); + if (GetValue(traceClass, "environment") is Dictionary environment) + { + foreach (var entry in environment) + { + environmentBag.AddValue(entry.Key, Convert.ToString(entry.Value, CultureInfo.InvariantCulture)); + } + } + + this.metadataBuilder.SetEnvironmentDescriptor(new CtfEnvironmentDescriptor(environmentBag)); + } + + private static CtfClockDescriptor CreateClock(Dictionary clockClass) + { + // CTF 2 data stream classes refer to clock classes by id; older drafts only had a name. + string clockName = GetString(clockClass, "id") ?? GetRequiredString(clockClass, "name"); + + ulong frequency = TryGetNumber(clockClass, "frequency", out var frequencyNumber) ? frequencyNumber.AsULong() : 1000000000ul; + + var bag = new CtfPropertyBag(); + bag.AddValue("name", clockName); + bag.AddValue("freq", frequency.ToString(CultureInfo.InvariantCulture)); + + string description = GetString(clockClass, "description"); + if (description != null) + { + bag.AddValue("description", description); + } + + string uid = GetString(clockClass, "uid") ?? FormatUuid(GetValue(clockClass, "uuid")); + if (uid != null && Guid.TryParse(uid, out _)) + { + bag.AddValue("uuid", uid); + } + + if (TryGetNumber(clockClass, "precision", out var precision) && !precision.IsNegative) + { + bag.AddValue("precision", precision.Text); + } + + if (GetValue(clockClass, "offset-from-origin") is Dictionary offset) + { + // The descriptor expects the full offset in clock cycles. + decimal cycles = 0; + if (TryGetNumber(offset, "seconds", out var seconds)) + { + cycles += decimal.Parse(seconds.Text, CultureInfo.InvariantCulture) * frequency; + } + + if (TryGetNumber(offset, "cycles", out var offsetCycles)) + { + cycles += decimal.Parse(offsetCycles.Text, CultureInfo.InvariantCulture); + } + + if (cycles > 0) + { + bag.AddValue("offset", decimal.ToUInt64(cycles).ToString(CultureInfo.InvariantCulture)); + } + } + + bag.AddValue("absolute", GetValue(clockClass, "origin") != null ? "true" : "false"); + + return new CtfClockDescriptor(bag); + } + + private void AddDataStreamClass(Dictionary dataStreamClass) + { + uint id = GetDataStreamClassId(dataStreamClass); + string clockName = GetString(dataStreamClass, "default-clock-class-id"); + + var packetContextClass = GetValue(dataStreamClass, "packet-context-field-class") + ?? throw new CtfMetadataException($"CTF 2 data stream class {id} does not define a packet context."); + var eventHeaderClass = GetValue(dataStreamClass, "event-record-header-field-class") + ?? throw new CtfMetadataException($"CTF 2 data stream class {id} does not define an event record header."); + + if (!(this.BuildFieldClass(packetContextClass, BuildContext.Root("packet-context", clockName, false)) is CtfStructDescriptor packetContext)) + { + throw new CtfMetadataException($"CTF 2 data stream class {id} packet context is not a structure."); + } + + if (!(this.BuildFieldClass(eventHeaderClass, BuildContext.Root("event-record-header", clockName, false)) is CtfStructDescriptor eventHeader)) + { + throw new CtfMetadataException($"CTF 2 data stream class {id} event record header is not a structure."); + } + + CtfStructDescriptor eventContext = null; + var eventContextClass = GetValue(dataStreamClass, "event-record-common-context-field-class"); + if (eventContextClass != null) + { + var commonContext = BuildContext.Root("event-record-common-context", clockName, this.prefixEventFieldNamesWithUnderscore); + eventContext = this.BuildFieldClass(eventContextClass, commonContext) as CtfStructDescriptor + ?? throw new CtfMetadataException($"CTF 2 data stream class {id} event record common context is not a structure."); + } + + var bag = new CtfPropertyBag(); + bag.AddValue("id", id.ToString(CultureInfo.InvariantCulture)); + + this.metadataBuilder.AddStream(new CtfStreamDescriptor(bag, eventHeader, eventContext, packetContext)); + } + + private void AddEventRecordClass(Dictionary eventRecordClass) + { + string name = GetString(eventRecordClass, "name") ?? string.Empty; + try + { + uint streamId = TryGetNumber(eventRecordClass, "data-stream-class-id", out var streamIdNumber) ? (uint)streamIdNumber.AsULong() : 0; + ulong id = TryGetNumber(eventRecordClass, "id", out var idNumber) ? idNumber.AsULong() : 0; + + var assignments = new Dictionary(StringComparer.Ordinal) + { + ["name"] = name, + ["id"] = id.ToString(CultureInfo.InvariantCulture), + ["stream_id"] = streamId.ToString(CultureInfo.InvariantCulture), + }; + + if (TryGetLogLevel(eventRecordClass, out string logLevel)) + { + assignments["loglevel"] = logLevel; + } + + var typeDeclarations = new Dictionary(StringComparer.Ordinal); + + var payloadClass = GetValue(eventRecordClass, "payload-field-class"); + typeDeclarations["fields"] = payloadClass != null + ? this.BuildFieldClass(payloadClass, BuildContext.Root("event-record-payload", null, this.prefixEventFieldNamesWithUnderscore)) + : new CtfStructDescriptor(new CtfPropertyBag(), Array.Empty()); + + var specificContextClass = GetValue(eventRecordClass, "specific-context-field-class"); + if (specificContextClass != null) + { + typeDeclarations["context"] = this.BuildFieldClass( + specificContextClass, + BuildContext.Root("event-record-specific-context", null, this.prefixEventFieldNamesWithUnderscore)); + } + + this.metadataBuilder.AddEvent(assignments, typeDeclarations); + } + catch (Exception e) when (!(e is CtfMetadataException)) + { + throw new CtfMetadataException($"Unable to process CTF 2 event record class '{name}': {e.Message}", e); + } + } + + private CtfMetadataTypeDescriptor BuildFieldClass(object fieldClass, BuildContext context, IReadOnlyList selectorOptions = null) + { + var fieldClassObject = this.ResolveFieldClass(fieldClass); + string type = GetRequiredString(fieldClassObject, "type"); + + switch (type) + { + case "fixed-length-unsigned-integer": + case "fixed-length-bit-array": + case "fixed-length-bit-map": + case "fixed-length-boolean": + return BuildFixedLengthInteger(fieldClassObject, false, context, selectorOptions); + + case "fixed-length-signed-integer": + return BuildFixedLengthInteger(fieldClassObject, true, context, selectorOptions); + + case "variable-length-unsigned-integer": + case "variable-length-signed-integer": + if (selectorOptions != null) + { + throw new CtfMetadataException("Variable-length integer variant selectors are not supported."); + } + + return new Ctf2VariableLengthIntegerDescriptor( + type == "variable-length-signed-integer", + GetDisplayBase(fieldClassObject), + GetClockMap(fieldClassObject, context)); + + case "fixed-length-floating-point-number": + case "fixed-length-floating-point": + return BuildFloatingPoint(fieldClassObject); + + case "null-terminated-string": + EnsureUtf8(fieldClassObject); + var stringBag = new CtfPropertyBag(); + stringBag.AddValue("encoding", "UTF8"); + return new CtfStringDescriptor(stringBag); + + case "static-length-string": + EnsureUtf8(fieldClassObject); + return new CtfArrayDescriptor(CreateByte("UTF8", 10), GetRequiredNumber(fieldClassObject, "length").AsULong().ToString(CultureInfo.InvariantCulture)); + + case "dynamic-length-string": + EnsureUtf8(fieldClassObject); + return new CtfArrayDescriptor(CreateByte("UTF8", 10), this.GetFieldLocation(fieldClassObject, "length-field-location", context)); + + case "static-length-blob": + return new CtfArrayDescriptor(CreateByte(null, 16), GetRequiredNumber(fieldClassObject, "length").AsULong().ToString(CultureInfo.InvariantCulture)); + + case "dynamic-length-blob": + return new CtfArrayDescriptor(CreateByte(null, 16), this.GetFieldLocation(fieldClassObject, "length-field-location", context)); + + case "static-length-array": + return new CtfArrayDescriptor( + this.BuildArrayElement(fieldClassObject, context), + GetRequiredNumber(fieldClassObject, "length").AsULong().ToString(CultureInfo.InvariantCulture)); + + case "dynamic-length-array": + return new CtfArrayDescriptor( + this.BuildArrayElement(fieldClassObject, context), + this.GetFieldLocation(fieldClassObject, "length-field-location", context)); + + case "structure": + return this.BuildStructure(fieldClassObject, context); + + case "variant": + return this.BuildVariant(fieldClassObject, context); + + default: + throw new CtfMetadataException($"CTF 2 field class type '{type}' is not supported."); + } + } + + private Dictionary ResolveFieldClass(object fieldClass) + { + var seen = new HashSet(StringComparer.Ordinal); + while (fieldClass is string aliasName) + { + if (!seen.Add(aliasName) || !this.aliases.TryGetValue(aliasName, out fieldClass)) + { + throw new CtfMetadataException($"Unknown CTF 2 field class alias '{aliasName}'."); + } + } + + return fieldClass as Dictionary + ?? throw new CtfMetadataException("CTF 2 field class is not a JSON object."); + } + + private static CtfMetadataTypeDescriptor BuildFixedLengthInteger( + Dictionary fieldClass, + bool signed, + BuildContext context, + IReadOnlyList selectorOptions) + { + // Like the CTF 1.8 path, only whole-byte big-endian integers are byte swapped when read; sub-byte + // big-endian bit fields (e.g. IPv4 header fields in lttng-modules net_* events) are still readable. + bool bigEndian = GetString(fieldClass, "byte-order") == "big-endian"; + var length = GetRequiredNumber(fieldClass, "length"); + + var bag = new CtfPropertyBag(); + bag.AddValue("size", length.Text); + bag.AddValue("align", GetAlignment(fieldClass, "alignment").ToString(CultureInfo.InvariantCulture)); + bag.AddValue("signed", signed ? "true" : "false"); + bag.AddValue("byte_order", bigEndian ? "be" : "le"); + bag.AddValue("base", GetDisplayBase(fieldClass).ToString(CultureInfo.InvariantCulture)); + + string map = GetClockMap(fieldClass, context); + if (map != null) + { + bag.AddValue("map", map); + } + + var integer = new CtfIntegerDescriptor(bag); + if (selectorOptions == null) + { + return integer; + } + + // Variant selection in this library is done through an enumeration whose labels are the option names. + var enumeration = new CtfEnumDescriptor(integer); + foreach (var option in selectorOptions) + { + foreach (var range in option.Ranges) + { + var begin = CreateIntegerLiteral(range.Item1, signed); + var end = CreateIntegerLiteral(range.Item2, signed); + if (!enumeration.AddRange(option.Name, new CtfIntegerRange(integer, begin, end))) + { + throw new CtfMetadataException($"Unable to add variant selector range for option '{option.Name}'."); + } + } + } + + return enumeration; + } + + private static CtfMetadataTypeDescriptor BuildFloatingPoint(Dictionary fieldClass) + { + var bag = new CtfPropertyBag(); + switch (GetRequiredNumber(fieldClass, "length").AsLong()) + { + case 32: + bag.AddValue("exp_dig", "8"); + bag.AddValue("mant_dig", "24"); + break; + case 64: + bag.AddValue("exp_dig", "11"); + bag.AddValue("mant_dig", "53"); + break; + default: + throw new CtfMetadataException("Only 32-bit and 64-bit CTF 2 floating point numbers are supported."); + } + + bag.AddValue("align", GetAlignment(fieldClass, "alignment").ToString(CultureInfo.InvariantCulture)); + bag.AddValue("byte_order", GetString(fieldClass, "byte-order") == "big-endian" ? "be" : "le"); + return new CtfFloatingPointDescriptor(bag); + } + + // Shared by static-length and dynamic-length arrays: both use only "element-field-class" and + // "minimum-alignment" here; the length ("length" or "length-field-location") is handled by the caller. + private CtfMetadataTypeDescriptor BuildArrayElement(Dictionary fieldClass, BuildContext context) + { + var element = this.BuildFieldClass(GetRequiredValue(fieldClass, "element-field-class"), context.NotAddressable()); + if (TryGetNumber(fieldClass, "minimum-alignment", out var minimumAlignment) && minimumAlignment.AsLong() > element.Align) + { + throw new CtfMetadataException("CTF 2 arrays with a minimum alignment greater than their element alignment are not supported."); + } + + return element; + } + + private CtfStructDescriptor BuildStructure(Dictionary fieldClass, BuildContext context) + { + var members = GetValue(fieldClass, "member-classes") as List ?? new List(); + var memberObjects = members.Select(member => member as Dictionary + ?? throw new CtfMetadataException("CTF 2 structure member class is not a JSON object.")).ToList(); + + // A variant selects its option through a sibling integer field; collect the option ranges so that + // the selector can be built as an enumeration. + var selectorOptionsByMember = new Dictionary>(StringComparer.Ordinal); + foreach (var member in memberObjects) + { + var memberClass = this.ResolveFieldClass(GetRequiredValue(member, "field-class")); + if (GetString(memberClass, "type") != "variant") + { + continue; + } + + string memberName = GetRequiredString(member, "name"); + string selectorName = GetSiblingSelectorName(memberClass, context.ForMember(memberName)); + if (!selectorOptionsByMember.TryGetValue(selectorName, out var selectorOptions)) + { + selectorOptions = new List(); + selectorOptionsByMember.Add(selectorName, selectorOptions); + } + + selectorOptions.AddRange(GetVariantOptions(memberClass, context)); + } + + var fields = new List(memberObjects.Count); + foreach (var member in memberObjects) + { + string memberName = GetRequiredString(member, "name"); + selectorOptionsByMember.TryGetValue(memberName, out var selectorOptions); + var memberType = this.BuildFieldClass(GetRequiredValue(member, "field-class"), context.ForMember(memberName), selectorOptions); + fields.Add(new CtfFieldDescriptor(memberType, context.MapName(memberName))); + } + + var bag = new CtfPropertyBag(); + bag.AddValue("align", GetAlignment(fieldClass, "minimum-alignment").ToString(CultureInfo.InvariantCulture)); + return new CtfStructDescriptor(bag, fields.ToArray()); + } + + private CtfVariantDescriptor BuildVariant(Dictionary fieldClass, BuildContext context) + { + string selectorName = context.MapName(GetSiblingSelectorName(fieldClass, context)); + + var options = GetValue(fieldClass, "options") as List + ?? throw new CtfMetadataException("CTF 2 variant does not define any options."); + + var fields = new List(options.Count); + for (int index = 0; index < options.Count; index++) + { + var option = options[index] as Dictionary + ?? throw new CtfMetadataException("CTF 2 variant option is not a JSON object."); + var optionType = this.BuildFieldClass(GetRequiredValue(option, "field-class"), context.NotAddressable()); + fields.Add(new CtfFieldDescriptor(optionType, GetOptionName(option, index, context))); + } + + return new CtfVariantDescriptor(selectorName, fields); + } + + private static IEnumerable GetVariantOptions(Dictionary variantClass, BuildContext context) + { + var options = GetValue(variantClass, "options") as List + ?? throw new CtfMetadataException("CTF 2 variant does not define any options."); + + for (int index = 0; index < options.Count; index++) + { + var option = (Dictionary)options[index]; + var ranges = GetValue(option, "selector-field-ranges") as List + ?? throw new CtfMetadataException("CTF 2 variant option does not define selector field ranges."); + + var optionRanges = new List>(); + foreach (var range in ranges) + { + if (!(range is List bounds) || bounds.Count != 2 || + !(bounds[0] is Ctf2JsonNumber lower) || !(bounds[1] is Ctf2JsonNumber upper)) + { + throw new CtfMetadataException("CTF 2 variant selector field range is invalid."); + } + + optionRanges.Add(Tuple.Create(lower, upper)); + } + + yield return new SelectorOption(GetOptionName(option, index, context), optionRanges); + } + } + + private static string GetOptionName(Dictionary option, int index, BuildContext context) + { + string name = GetString(option, "name"); + return name == null ? $"option{index}" : context.MapName(name); + } + + private static string GetSiblingSelectorName(Dictionary variantClass, BuildContext context) + { + var path = ResolveFieldLocation(variantClass, "selector-field-location", context); + if (path.Count != 1) + { + throw new CtfMetadataException("Only CTF 2 variants whose selector is a sibling field are supported."); + } + + return path[0]; + } + + private string GetFieldLocation(Dictionary fieldClass, string property, BuildContext context) + { + return string.Join(".", ResolveFieldLocation(fieldClass, property, context).Select(context.MapName)); + } + + /// + /// Returns the path of a field location (https://diamon.org/ctf/#field-loc) relative to the structure that + /// contains the field being built, which is how playback resolves it. + /// + private static List ResolveFieldLocation(Dictionary fieldClass, string property, BuildContext context) + { + if (!(GetValue(fieldClass, property) is Dictionary location) || + !(GetValue(location, "path") is List pathElements) || + pathElements.Count == 0) + { + throw new CtfMetadataException($"CTF 2 field class has an invalid '{property}'."); + } + + var path = pathElements.Select(element => element as string + ?? throw new CtfMetadataException($"CTF 2 '{property}' paths to a parent structure (null elements) are not supported.")).ToList(); + + string origin = GetString(location, "origin"); + if (origin == null) + { + // Without an origin, the path starts at the structure containing the field. + return path; + } + + // With an origin, the path starts at the root structure of that scope. LTTng-UST metadata writes all its + // locations this way, with the target in the same structure as the field. Such a path is made relative by + // removing the containing structure's own path. Targets in another scope, or outside the containing + // structure, would need lookups that playback doesn't support. + var container = context.Path?.Take(context.Path.Count - 1).ToList(); + if (!StringComparer.Ordinal.Equals(origin, context.Scope) || + container == null || + path.Count <= container.Count || + !path.Take(container.Count).SequenceEqual(container, StringComparer.Ordinal)) + { + throw new CtfMetadataException( + $"CTF 2 '{property}' with origin '{origin}' and path [{string.Join(", ", path)}] is not supported: " + + $"the target must be within the '{context.Scope}' structure that contains the field."); + } + + return path.Skip(container.Count).ToList(); + } + + private static CtfIntegerDescriptor CreateByte(string encoding, int displayBase) + { + var bag = new CtfPropertyBag(); + bag.AddValue("size", "8"); + bag.AddValue("align", "8"); + bag.AddValue("signed", "false"); + bag.AddValue("byte_order", "le"); + bag.AddValue("base", displayBase.ToString(CultureInfo.InvariantCulture)); + if (encoding != null) + { + bag.AddValue("encoding", encoding); + } + + return new CtfIntegerDescriptor(bag); + } + + private static string GetClockMap(Dictionary fieldClass, BuildContext context) + { + if (context.ClockName == null || !(GetValue(fieldClass, "roles") is List roles)) + { + return null; + } + + bool isTimestamp = roles.OfType().Any(role => + role == DefaultClockTimestampRole || role == PacketEndDefaultClockTimestampRole); + + return isTimestamp ? $"clock.{context.ClockName}.value" : null; + } + + private static int GetDisplayBase(Dictionary fieldClass) + { + return TryGetNumber(fieldClass, "preferred-display-base", out var displayBase) ? (int)displayBase.AsLong() : 10; + } + + private static long GetAlignment(Dictionary fieldClass, string property) + { + return TryGetNumber(fieldClass, property, out var alignment) ? alignment.AsLong() : 1; + } + + private static void EnsureUtf8(Dictionary fieldClass) + { + string encoding = GetString(fieldClass, "encoding"); + if (encoding != null && encoding != "utf-8") + { + throw new CtfMetadataException($"CTF 2 string encoding '{encoding}' is not supported."); + } + } + + private static IntegerLiteral CreateIntegerLiteral(Ctf2JsonNumber number, bool signed) + { + return signed ? new IntegerLiteral(number.AsLong()) : new IntegerLiteral(number.AsULong()); + } + + private static bool TryGetLogLevel(Dictionary eventRecordClass, out string logLevel) + { + logLevel = null; + if (!(GetValue(eventRecordClass, "attributes") is Dictionary attributes)) + { + return false; + } + + foreach (var namespacedAttributes in attributes.Values.OfType>()) + { + if (TryGetNumber(namespacedAttributes, "log-level", out var level) && !level.IsNegative) + { + logLevel = level.Text; + return true; + } + } + + return false; + } + + private static uint GetDataStreamClassId(Dictionary dataStreamClass) + { + return TryGetNumber(dataStreamClass, "id", out var id) ? (uint)id.AsULong() : 0; + } + + private static string FormatUuid(object uuid) + { + if (!(uuid is List bytes) || bytes.Count != 16 || !bytes.All(b => b is Ctf2JsonNumber)) + { + return null; + } + + var hex = string.Concat(bytes.Cast().Select(b => b.AsULong().ToString("x2", CultureInfo.InvariantCulture))); + return $"{hex.Substring(0, 8)}-{hex.Substring(8, 4)}-{hex.Substring(12, 4)}-{hex.Substring(16, 4)}-{hex.Substring(20)}"; + } + + private static object GetValue(Dictionary json, string property) + { + return json.TryGetValue(property, out var value) ? value : null; + } + + private static object GetRequiredValue(Dictionary json, string property) + { + return GetValue(json, property) ?? throw new CtfMetadataException($"CTF 2 metadata is missing the required '{property}' property."); + } + + private static string GetString(Dictionary json, string property) + { + return GetValue(json, property) as string; + } + + private static string GetRequiredString(Dictionary json, string property) + { + return GetString(json, property) ?? throw new CtfMetadataException($"CTF 2 metadata is missing the required '{property}' string property."); + } + + private static bool TryGetNumber(Dictionary json, string property, out Ctf2JsonNumber number) + { + number = GetValue(json, property) as Ctf2JsonNumber; + return number != null; + } + + private static Ctf2JsonNumber GetRequiredNumber(Dictionary json, string property) + { + return TryGetNumber(json, property, out var number) + ? number + : throw new CtfMetadataException($"CTF 2 metadata is missing the required '{property}' number property."); + } + + private sealed class BuildContext + { + private BuildContext(string scope, string clockName, bool prefixNames, IReadOnlyList path) + { + this.Scope = scope; + this.ClockName = clockName; + this.PrefixNames = prefixNames; + this.Path = path; + } + + /// + /// The scope being built, named like a field location origin (e.g. "event-record-payload"). + /// + internal string Scope { get; } + + internal string ClockName { get; } + + internal bool PrefixNames { get; } + + /// + /// Member names from the scope's root structure to the field class being built, or null when the field + /// can't be reached through structure members (array elements and variant options). + /// + internal IReadOnlyList Path { get; } + + internal static BuildContext Root(string scope, string clockName, bool prefixNames) => + new BuildContext(scope, clockName, prefixNames, Array.Empty()); + + internal BuildContext ForMember(string name) => + new BuildContext(this.Scope, this.ClockName, this.PrefixNames, this.Path?.Concat(new[] { name }).ToArray()); + + internal BuildContext NotAddressable() => + new BuildContext(this.Scope, this.ClockName, this.PrefixNames, null); + + internal string MapName(string name) => this.PrefixNames ? "_" + name : name; + } + + private sealed class SelectorOption + { + internal SelectorOption(string name, IReadOnlyList> ranges) + { + this.Name = name; + this.Ranges = ranges; + } + + internal string Name { get; } + + internal IReadOnlyList> Ranges { get; } + } + } +} diff --git a/CtfPlayback/Metadata/Ctf2/Ctf2VariableLengthIntegerDescriptor.cs b/CtfPlayback/Metadata/Ctf2/Ctf2VariableLengthIntegerDescriptor.cs new file mode 100644 index 0000000..a9f9400 --- /dev/null +++ b/CtfPlayback/Metadata/Ctf2/Ctf2VariableLengthIntegerDescriptor.cs @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using CtfPlayback.FieldValues; +using CtfPlayback.Helpers; +using CtfPlayback.Metadata.Helpers; +using CtfPlayback.Metadata.TypeInterfaces; +using CtfPlayback.Metadata.Types; + +namespace CtfPlayback.Metadata.Ctf2 +{ + /// + /// CTF 2 variable-length (unsigned or signed LEB128) integer. + /// + internal sealed class Ctf2VariableLengthIntegerDescriptor + : CtfMetadataTypeDescriptor, + ICtfIntegerDescriptor + { + internal Ctf2VariableLengthIntegerDescriptor(bool signed, int displayBase, string map) + : base(CtfTypes.Integer) + { + this.Signed = signed; + this.Base = (short)displayBase; + this.Map = map; + } + + /// + public int Size => 64; + + /// + public bool Signed { get; } + + /// + public string Encoding => "none"; + + /// + public short Base { get; } + + /// + public string Map { get; } + + /// + public override int Align => 8; + + /// + public override CtfFieldValue Read(IPacketReader reader, CtfFieldValue parent = null) + { + Guard.NotNull(reader, nameof(reader)); + + reader.Align((uint)this.Align); + + return new CtfIntegerValue(DecodeLeb128(reader, this.Signed), this); + } + + /// + /// Decodes an unsigned or signed LEB128 value (https://en.wikipedia.org/wiki/LEB128): each byte holds 7 bits + /// of the value, least significant group first, and its high bit is set when more bytes follow. A signed + /// value is sign-extended from bit 6 of its last byte. Bits beyond 64 are ignored. + /// + internal static IntegerLiteral DecodeLeb128(IPacketReader reader, bool signed) + { + ulong value = 0; + int shift = 0; + byte current; + do + { + current = reader.ReadBits(8)[0]; + if (shift < 64) + { + value |= (ulong)(current & 0x7f) << shift; + } + + shift += 7; + } + while ((current & 0x80) != 0); + + if (!signed) + { + return new IntegerLiteral(value); + } + + if (shift < 64 && (current & 0x40) != 0) + { + value |= ulong.MaxValue << shift; + } + + return new IntegerLiteral((long)value); + } + + /// + public override string ToString() + { + return this.Signed ? "vlint" : "vluint"; + } + } +} diff --git a/CtfPlayback/Metadata/CtfMetadataExtensions.cs b/CtfPlayback/Metadata/CtfMetadataExtensions.cs new file mode 100644 index 0000000..9cbdecf --- /dev/null +++ b/CtfPlayback/Metadata/CtfMetadataExtensions.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using CtfPlayback.Metadata.Interfaces; + +namespace CtfPlayback.Metadata +{ + internal static class CtfMetadataExtensions + { + /// + /// Finds the stream descriptor with the given id. Stream ids are usually also their index, but + /// this is not required (e.g. CTF 2 metadata may declare streams in any order). + /// + internal static ICtfStreamDescriptor GetStream(this ICtfMetadata metadata, uint streamId) + { + var streams = metadata.Streams; + if (streamId < streams.Count && streams[(int)streamId].Id == streamId) + { + return streams[(int)streamId]; + } + + foreach (var stream in streams) + { + if (stream.Id == streamId) + { + return stream; + } + } + + throw new CtfPlaybackException($"The metadata does not describe stream id {streamId}."); + } + } +} diff --git a/CtfPlayback/Metadata/CtfMetadataText.cs b/CtfPlayback/Metadata/CtfMetadataText.cs new file mode 100644 index 0000000..d5a8770 --- /dev/null +++ b/CtfPlayback/Metadata/CtfMetadataText.cs @@ -0,0 +1,139 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Buffers.Binary; +using System.IO; +using System.Text; + +namespace CtfPlayback.Metadata +{ + /// + /// Helpers for reading a metadata stream, which may be plain text or packetized + /// (CTF 1.8: https://diamon.org/ctf/v1.8.3/#spec7.1, CTF 2: https://diamon.org/ctf/#metadata-stream and + /// https://diamon.org/ctf/files/CTF2-PMETA-1.0.html). + /// + internal static class CtfMetadataText + { + // Magic number of a metadata packet, defined by CTF 1.8 section 7.1 (https://diamon.org/ctf/v1.8.3/#spec7.1) + // and CTF2-PMETA-1.0 (https://diamon.org/ctf/files/CTF2-PMETA-1.0.html). It may be stored in either byte order. + // This is not the magic number of data stream packets (0xC1FC1FC1). + private const uint MetadataPacketMagic = 0x75d11d57; + + // CTF 1.8: magic(4) + uuid(16) + checksum(4) + content_size(4) + packet_size(4) + 5 single-byte fields. + // LTTng (2.15 and later) also uses this header for CTF 2 metadata, with version 2.0. + private const int Ctf1PacketHeaderSize = 37; + + // CTF2-PMETA-1.0: the CTF 1.8 fields, 3 reserved bytes, then the header size in bits (352 = 44 bytes). + private const int Ctf2MinimumPacketHeaderSize = 44; + + private const int ContentSizeOffset = 24; + private const int PacketSizeOffset = 28; + private const int MajorVersionOffset = 35; + private const int Ctf2ReservedOffset = 37; + private const int Ctf2HeaderSizeOffset = 40; + + internal static byte[] ReadAllBytes(Stream stream) + { + using (var memoryStream = new MemoryStream()) + { + stream.CopyTo(memoryStream); + return memoryStream.ToArray(); + } + } + + /// + /// Returns the metadata text, removing packet headers if the metadata is packetized. + /// + internal static string Read(byte[] metadata) + { + if (metadata.Length < Ctf1PacketHeaderSize || !TryGetPacketByteOrder(metadata, 0, out _)) + { + return Encoding.UTF8.GetString(metadata); + } + + // Content is decoded once at the end because a multi-byte UTF-8 character may span two packets. + var content = new MemoryStream(metadata.Length); + long offset = 0; + while (offset + Ctf1PacketHeaderSize <= metadata.Length) + { + int packetOffset = (int)offset; + + // The encoding of the magic number gives the byte order of the header's other integer fields. + if (!TryGetPacketByteOrder(metadata, packetOffset, out bool bigEndian)) + { + throw new InvalidDataException("Metadata stream seems to be corrupt: invalid packet magic number."); + } + + uint contentBits = ReadUInt32(metadata, packetOffset + ContentSizeOffset, bigEndian); + uint packetBits = ReadUInt32(metadata, packetOffset + PacketSizeOffset, bigEndian); + if ((contentBits % 8) != 0 || (packetBits % 8) != 0) + { + throw new InvalidDataException("Metadata stream seems to be corrupt: packet size is not a whole number of bytes."); + } + + long contentBytes = contentBits / 8; + long packetBytes = packetBits / 8; + int headerBytes = GetHeaderSize(metadata, packetOffset, bigEndian, contentBytes); + if (contentBytes < headerBytes || packetBytes < contentBytes || + offset + contentBytes > metadata.Length || offset + packetBytes > metadata.Length) + { + throw new InvalidDataException("Metadata stream seems to be corrupt: invalid packet size."); + } + + content.Write(metadata, packetOffset + headerBytes, (int)contentBytes - headerBytes); + offset += packetBytes; + } + + if (offset != metadata.Length) + { + throw new InvalidDataException("Metadata stream seems to be corrupt: truncated packet header."); + } + + return Encoding.UTF8.GetString(content.GetBuffer(), 0, (int)content.Length); + } + + private static int GetHeaderSize(byte[] metadata, int packetOffset, bool bigEndian, long contentBytes) + { + // Version 1.8 packets, and LTTng's version 2.0 packets, have the 37-byte CTF 1.8 header. Only version 2.0 + // packets with enough content can have the longer CTF2-PMETA-1.0 header. + if (metadata[packetOffset + MajorVersionOffset] < 2 || + contentBytes < Ctf2MinimumPacketHeaderSize || + packetOffset + Ctf2MinimumPacketHeaderSize > metadata.Length) + { + return Ctf1PacketHeaderSize; + } + + uint headerBits = ReadUInt32(metadata, packetOffset + Ctf2HeaderSizeOffset, bigEndian); + if ((headerBits % 8) == 0 && headerBits >= Ctf2MinimumPacketHeaderSize * 8 && headerBits / 8 <= contentBytes) + { + return (int)(headerBits / 8); + } + + // In LTTng's layout these bytes are metadata text. Text never contains NUL bytes, so zero "reserved" bytes + // mean a CTF2-PMETA-1.0 header with a bad size. (Every text byte is at least 0x09, so text read as a header + // size is over 18 MB and can't pass the check above for a real metadata packet.) + if (metadata[packetOffset + Ctf2ReservedOffset] == 0 && + metadata[packetOffset + Ctf2ReservedOffset + 1] == 0 && + metadata[packetOffset + Ctf2ReservedOffset + 2] == 0) + { + throw new InvalidDataException("Metadata stream seems to be corrupt: invalid packet header size."); + } + + return Ctf1PacketHeaderSize; + } + + private static bool TryGetPacketByteOrder(byte[] metadata, int packetOffset, out bool bigEndian) + { + var magic = new ReadOnlySpan(metadata, packetOffset, sizeof(uint)); + bigEndian = BinaryPrimitives.ReadUInt32BigEndian(magic) == MetadataPacketMagic; + return bigEndian || BinaryPrimitives.ReadUInt32LittleEndian(magic) == MetadataPacketMagic; + } + + private static uint ReadUInt32(byte[] metadata, int index, bool bigEndian) + { + var bytes = new ReadOnlySpan(metadata, index, sizeof(uint)); + return bigEndian ? BinaryPrimitives.ReadUInt32BigEndian(bytes) : BinaryPrimitives.ReadUInt32LittleEndian(bytes); + } + } +} diff --git a/CtfPlayback/Metadata/CtfVersionDetectingMetadataParser.cs b/CtfPlayback/Metadata/CtfVersionDetectingMetadataParser.cs new file mode 100644 index 0000000..811f6cc --- /dev/null +++ b/CtfPlayback/Metadata/CtfVersionDetectingMetadataParser.cs @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.IO; +using CtfPlayback.Metadata.AntlrParser; +using CtfPlayback.Metadata.Ctf2; +using CtfPlayback.Metadata.Interfaces; + +namespace CtfPlayback.Metadata +{ + /// + /// Parses either CTF 1.8 (TSDL) or CTF 2 (JSON) metadata, based on the content of the metadata stream. + /// + public sealed class CtfVersionDetectingMetadataParser + : ICtfMetadataParser + { + private readonly ICtfMetadataCustomization metadataCustomization; + private readonly ICtfMetadataBuilder metadataBuilder; + private readonly bool prefixCtf2EventFieldNamesWithUnderscore; + + /// + /// Constructor. + /// + /// Extension points for parsing metadata + /// The object used to build metadata + /// + /// See . + /// + public CtfVersionDetectingMetadataParser( + ICtfMetadataCustomization metadataCustomization, + ICtfMetadataBuilder metadataBuilder, + bool prefixCtf2EventFieldNamesWithUnderscore) + { + this.metadataCustomization = metadataCustomization; + this.metadataBuilder = metadataBuilder ?? throw new ArgumentNullException(nameof(metadataBuilder)); + this.prefixCtf2EventFieldNamesWithUnderscore = prefixCtf2EventFieldNamesWithUnderscore; + } + + /// + public ICtfMetadata Parse(Stream metadataStream) + { + if (metadataStream == null) + { + throw new ArgumentNullException(nameof(metadataStream)); + } + + byte[] metadata = CtfMetadataText.ReadAllBytes(metadataStream); + + // Packetized metadata (in either byte order) is decoded here, for both CTF 1.8 and CTF 2. + string text = CtfMetadataText.Read(metadata); + + if (Ctf2MetadataParser.IsCtf2Metadata(text)) + { + return new Ctf2MetadataParser(this.metadataBuilder, this.prefixCtf2EventFieldNamesWithUnderscore).Parse(text); + } + + return new CtfAntlrMetadataParser(this.metadataCustomization, this.metadataBuilder).ParseText(text); + } + } +} diff --git a/CtfPlayback/Metadata/Helpers/IntegerLiteral.cs b/CtfPlayback/Metadata/Helpers/IntegerLiteral.cs index 7fbb77d..68587df 100644 --- a/CtfPlayback/Metadata/Helpers/IntegerLiteral.cs +++ b/CtfPlayback/Metadata/Helpers/IntegerLiteral.cs @@ -587,6 +587,12 @@ private static ushort HighestUsedBit(long value) ushort highestUsedBit = 0; + // Math.Abs cannot represent the magnitude of long.MinValue, which needs all 64 bits (63 plus the sign). + if (value == long.MinValue) + { + return 63; + } + // a negative number would always use the most significant bit value = Math.Abs(value); diff --git a/CtfPlayback/Metadata/Types/CtfFloatingPointDescriptor.cs b/CtfPlayback/Metadata/Types/CtfFloatingPointDescriptor.cs index 22a1f7e..f3a7d5b 100644 --- a/CtfPlayback/Metadata/Types/CtfFloatingPointDescriptor.cs +++ b/CtfPlayback/Metadata/Types/CtfFloatingPointDescriptor.cs @@ -31,6 +31,9 @@ internal CtfFloatingPointDescriptor(CtfPropertyBag bag) /// public string ByteOrder { get; } + private bool IsExplicitlyBigEndian => + StringComparer.Ordinal.Equals(this.ByteOrder, "be") || StringComparer.Ordinal.Equals(this.ByteOrder, "network"); + /// exp_dig is the number of digits represented in the exponent. public int Exponent { get; } @@ -72,6 +75,18 @@ internal CtfFieldValue Read(byte[] buffer, int byteCount) Debug.Assert(byteCount > 0); Debug.Assert(byteCount <= 8); // Up to 64-bits + // Values are decoded as little endian; reverse explicitly big-endian values first. + if (this.IsExplicitlyBigEndian) + { + var littleEndianBuffer = new byte[byteCount]; + for (int x = 0; x < byteCount; x++) + { + littleEndianBuffer[x] = buffer[byteCount - 1 - x]; + } + + buffer = littleEndianBuffer; + } + if (byteCount == 4) { var bufferAsInt = BitConverter.ToInt32(buffer); diff --git a/CtfPlayback/Metadata/Types/CtfIntegerDescriptor.cs b/CtfPlayback/Metadata/Types/CtfIntegerDescriptor.cs index b7f5540..2be1566 100644 --- a/CtfPlayback/Metadata/Types/CtfIntegerDescriptor.cs +++ b/CtfPlayback/Metadata/Types/CtfIntegerDescriptor.cs @@ -62,6 +62,9 @@ internal CtfIntegerDescriptor(CtfPropertyBag bag) internal string ByteOrder { get; private set; } + private bool IsExplicitlyBigEndian => + StringComparer.Ordinal.Equals(this.ByteOrder, "be") || StringComparer.Ordinal.Equals(this.ByteOrder, "network"); + /// public override string ToString() { @@ -98,6 +101,17 @@ internal CtfFieldValue Read(byte[] buffer, int byteCount) // if byte order is not set, or if the value is "native", then endianness is determined by the trace descriptor // if the byte order is "be" or "network", then it is big endian // if the byte order is "le", then it is little endian + // Only explicitly big endian, whole-byte integers are handled for now (e.g. IPv4 addresses in network order). + if (this.IsExplicitlyBigEndian && (this.Size % 8) == 0) + { + var littleEndianBuffer = new byte[byteCount]; + for (int x = 0; x < byteCount; x++) + { + littleEndianBuffer[x] = buffer[byteCount - 1 - x]; + } + + buffer = littleEndianBuffer; + } IntegerLiteral value; @@ -162,12 +176,16 @@ private IntegerLiteral ReadSignedLittleEndianValue(byte[] buffer, int byteCount) value |= buffer[x]; } - int signedMask = 1 << (this.Size - 1); - if ((value & signedMask) != 0) + // The sign bit must be computed in 64 bits: an int shift wraps for sizes above 32 bits. A 64-bit value + // already has its sign in place. + if (this.Size < 64) { - // extend the high order signed bit - long mask = ~(signedMask - 1); - value = value | mask; + long signBit = 1L << (this.Size - 1); + if ((value & signBit) != 0) + { + // extend the high order signed bit + value |= ~(signBit - 1); + } } return new IntegerLiteral(value); diff --git a/CtfUnitTest/Ctf2MetadataTests.cs b/CtfUnitTest/Ctf2MetadataTests.cs new file mode 100644 index 0000000..14caa34 --- /dev/null +++ b/CtfUnitTest/Ctf2MetadataTests.cs @@ -0,0 +1,441 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using CtfPlayback; +using CtfPlayback.FieldValues; +using CtfPlayback.Metadata; +using CtfPlayback.Metadata.Ctf2; +using CtfPlayback.Metadata.Interfaces; +using CtfPlayback.Metadata.InternalHelpers; +using CtfPlayback.Metadata.TypeInterfaces; +using CtfPlayback.Metadata.Types; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace CtfUnitTest +{ + [TestClass] + public class Ctf2MetadataTests + { + // Modeled on the metadata written by LTTng 2.16 for a kernel trace with perf counter contexts. + private static readonly string[] LTTngKernelFragments = + { + @"{""type"":""preamble"",""version"":2,""uuid"":[55,65,178,152,114,22,242,74,147,79,201,46,48,147,240,64]}", + @"{""type"":""field-class-alias"",""name"":""u32-erc-id"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":32,""alignment"":8,""byte-order"":""little-endian"",""roles"":[""event-record-class-id""]}}", + @"{""type"":""field-class-alias"",""name"":""u64-ts"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":64,""alignment"":8,""byte-order"":""little-endian"",""roles"":[""default-clock-timestamp""]}}", + @"{""type"":""field-class-alias"",""name"":""er-header-compact"",""field-class"":{""type"":""structure"",""minimum-alignment"":8,""member-classes"":[{""name"":""id"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":5,""byte-order"":""little-endian"",""mappings"":{""compact"":[[0,30]],""extended"":[[31,31]]},""roles"":[""event-record-class-id""]}},{""name"":""v"",""field-class"":{""type"":""variant"",""selector-field-location"":{""path"":[""id""]},""options"":[{""name"":""compact"",""selector-field-ranges"":[[0,30]],""field-class"":{""type"":""structure"",""member-classes"":[{""name"":""timestamp"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":27,""byte-order"":""little-endian"",""roles"":[""default-clock-timestamp""]}}]}},{""name"":""extended"",""selector-field-ranges"":[[31,31]],""field-class"":{""type"":""structure"",""member-classes"":[{""name"":""id"",""field-class"":""u32-erc-id""},{""name"":""timestamp"",""field-class"":""u64-ts""}]}}]}}]}}", + @"{""type"":""field-class-alias"",""name"":""pkt-ctx"",""field-class"":{""type"":""structure"",""member-classes"":[{""name"":""timestamp_begin"",""field-class"":""u64-ts""},{""name"":""timestamp_end"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":64,""alignment"":8,""byte-order"":""little-endian"",""roles"":[""packet-end-default-clock-timestamp""]}},{""name"":""content_size"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":64,""alignment"":8,""byte-order"":""little-endian""}},{""name"":""packet_size"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":64,""alignment"":8,""byte-order"":""little-endian""}},{""name"":""events_discarded"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":64,""alignment"":8,""byte-order"":""little-endian""}},{""name"":""cpu_id"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":32,""alignment"":8,""byte-order"":""little-endian""}}]}}", + @"{""type"":""field-class-alias"",""name"":""pkt-header"",""field-class"":{""type"":""structure"",""member-classes"":[{""name"":""magic"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":32,""alignment"":8,""byte-order"":""little-endian""}},{""name"":""uuid"",""field-class"":{""type"":""static-length-blob"",""length"":16}},{""name"":""stream_id"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":32,""alignment"":8,""byte-order"":""little-endian""}}]}}", + @"{""type"":""trace-class"",""uid"":""3741b298-7216-f24a-934f-c92e3093f040"",""environment"":{""domain"":""kernel"",""tracer_major"":2,""hostname"":""host""},""packet-header-field-class"":""pkt-header""}", + @"{""type"":""clock-class"",""id"":""monotonic"",""name"":""monotonic"",""frequency"":1000000000,""origin"":""unix-epoch"",""offset-from-origin"":{""seconds"":10,""cycles"":5}}", + @"{""type"":""data-stream-class"",""id"":1,""default-clock-class-id"":""monotonic"",""packet-context-field-class"":""pkt-ctx"",""event-record-header-field-class"":""er-header-compact"",""event-record-common-context-field-class"":{""type"":""structure"",""member-classes"":[{""name"":""tid"",""field-class"":{""type"":""fixed-length-signed-integer"",""length"":32,""byte-order"":""little-endian"",""alignment"":8}}]}}", + @"{""type"":""data-stream-class"",""id"":0,""default-clock-class-id"":""monotonic"",""packet-context-field-class"":""pkt-ctx"",""event-record-header-field-class"":""er-header-compact"",""event-record-common-context-field-class"":{""type"":""structure"",""member-classes"":[{""name"":""perf_cpu_cycles"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":64,""byte-order"":""little-endian"",""alignment"":8}},{""name"":""_callstack_kernel_length"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":32,""byte-order"":""little-endian"",""alignment"":8}},{""name"":""callstack_kernel"",""field-class"":{""type"":""dynamic-length-array"",""element-field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":64,""byte-order"":""little-endian"",""alignment"":8,""preferred-display-base"":16},""length-field-location"":{""path"":[""_callstack_kernel_length""]}}}]}}", + @"{""type"":""event-record-class"",""data-stream-class-id"":1,""id"":0,""name"":""sched_wakeup"",""payload-field-class"":{""type"":""structure"",""member-classes"":[{""name"":""comm"",""field-class"":{""type"":""static-length-string"",""length"":16}},{""name"":""tid"",""field-class"":{""type"":""fixed-length-signed-integer"",""length"":32,""byte-order"":""little-endian"",""alignment"":8}}]}}", + @"{""type"":""event-record-class"",""data-stream-class-id"":0,""id"":0,""name"":""sched_switch"",""attributes"":{""lttng.org,2009"":{""log-level"":13}},""payload-field-class"":{""type"":""structure"",""member-classes"":[{""name"":""prev_tid"",""field-class"":{""type"":""fixed-length-signed-integer"",""length"":32,""byte-order"":""little-endian"",""alignment"":8}},{""name"":""filename"",""field-class"":{""type"":""null-terminated-string""}},{""name"":""address_ipv4"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":32,""byte-order"":""big-endian"",""alignment"":8,""preferred-display-base"":16}},{""name"":""ihl"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":4,""byte-order"":""big-endian""}},{""name"":""ratio"",""field-class"":{""type"":""fixed-length-floating-point-number"",""length"":64,""byte-order"":""little-endian"",""alignment"":8}},{""name"":""ratio_be"",""field-class"":{""type"":""fixed-length-floating-point-number"",""length"":32,""byte-order"":""big-endian"",""alignment"":8}}]}}", + }; + + private static string LTTngKernelMetadata => string.Concat(LTTngKernelFragments.Select(fragment => "\u001e" + fragment + "\n")); + + [TestMethod] + public void DetectsCtf2Metadata() + { + Assert.IsTrue(Ctf2MetadataParser.IsCtf2Metadata(LTTngKernelMetadata)); + Assert.IsTrue(Ctf2MetadataParser.IsCtf2Metadata("\n \u001e{}")); + Assert.IsFalse(Ctf2MetadataParser.IsCtf2Metadata("/* CTF 1.8 */ trace { major = 1; };")); + Assert.IsFalse(Ctf2MetadataParser.IsCtf2Metadata(string.Empty)); + } + + [TestMethod] + public void ParsesTraceClockAndStreams() + { + var builder = Parse(LTTngKernelMetadata); + + Assert.AreEqual(Guid.Parse("3741b298-7216-f24a-934f-c92e3093f040"), builder.TraceDescriptor.Uuid); + Assert.AreEqual("kernel", builder.EnvironmentDescriptor.Properties["domain"]); + Assert.AreEqual("2", builder.EnvironmentDescriptor.Properties["tracer_major"]); + CollectionAssert.AreEqual( + new[] { "magic", "uuid", "stream_id" }, + builder.TraceDescriptor.PacketHeader.Fields.Select(f => f.Name).ToArray()); + + var clock = builder.ClocksByName["monotonic"]; + Assert.AreEqual(1000000000ul, clock.Frequency); + Assert.AreEqual(10000000005ul, clock.Offset); + + // Streams are declared out of order in the metadata but are stored by id. + CollectionAssert.AreEqual(new uint[] { 0, 1 }, builder.Streams.Select(s => s.Id).ToArray()); + } + + [TestMethod] + public void EventHeaderVariantSelectorIsAnEnumeration() + { + var builder = Parse(LTTngKernelMetadata); + var header = builder.Streams[0].EventHeader; + + var id = header.GetField("id").TypeDescriptor as ICtfEnumDescriptor; + Assert.IsNotNull(id); + Assert.AreEqual("compact", id.GetName(5ul)); + Assert.AreEqual("extended", id.GetName(31ul)); + + var variant = header.GetField("v").TypeDescriptor as ICtfVariantDescriptor; + Assert.IsNotNull(variant); + Assert.AreEqual("id", variant.Switch); + + var compactTimestamp = ((ICtfStructDescriptor)variant.GetVariant("compact").TypeDescriptor).GetField("timestamp").TypeDescriptor as ICtfIntegerDescriptor; + Assert.AreEqual(27, compactTimestamp.Size); + Assert.AreEqual("clock.monotonic.value", compactTimestamp.Map); + } + + [TestMethod] + public void EventFieldNamesArePrefixedLikeCtf18() + { + var builder = Parse(LTTngKernelMetadata); + + var context = builder.Streams[0].EventContext; + CollectionAssert.AreEqual( + new[] { "_perf_cpu_cycles", "__callstack_kernel_length", "_callstack_kernel" }, + context.Fields.Select(f => f.Name).ToArray()); + + var callstack = context.GetField("_callstack_kernel").TypeDescriptor as ICtfArrayDescriptor; + Assert.AreEqual("__callstack_kernel_length", callstack.Index); + + // Header and packet scopes keep their names. + Assert.IsNotNull(builder.Streams[0].PacketContext.GetField("timestamp_begin")); + } + + [TestMethod] + public void EventsAreIdentifiedByStreamAndId() + { + var builder = Parse(LTTngKernelMetadata); + + Assert.AreEqual(2, builder.AddedEvents.Count); + + var sched_switch = builder.AddedEvents.Single(e => e.Assignments["name"] == "sched_switch"); + Assert.AreEqual("0", sched_switch.Assignments["stream_id"]); + Assert.AreEqual("0", sched_switch.Assignments["id"]); + Assert.AreEqual("13", sched_switch.Assignments["loglevel"]); + + var sched_wakeup = builder.AddedEvents.Single(e => e.Assignments["name"] == "sched_wakeup"); + Assert.AreEqual("1", sched_wakeup.Assignments["stream_id"]); + Assert.AreEqual("0", sched_wakeup.Assignments["id"]); + + var payload = (ICtfStructDescriptor)sched_wakeup.TypeDeclarations["fields"]; + var comm = payload.GetField("_comm").TypeDescriptor as ICtfArrayDescriptor; + Assert.IsNotNull(comm); + Assert.AreEqual("16", comm.Index); + Assert.AreEqual("UTF8", ((ICtfIntegerDescriptor)comm.Type).Encoding); + + var switchPayload = (ICtfStructDescriptor)sched_switch.TypeDeclarations["fields"]; + Assert.IsInstanceOfType(switchPayload.GetField("_filename").TypeDescriptor, typeof(ICtfStringDescriptor)); + Assert.AreEqual(4, ((ICtfIntegerDescriptor)switchPayload.GetField("_ihl").TypeDescriptor).Size); + + var ratio = switchPayload.GetField("_ratio").TypeDescriptor as ICtfFloatingPointDescriptor; + Assert.IsNotNull(ratio); + Assert.AreEqual(11, ratio.Exponent); + Assert.AreEqual(53, ratio.Mantissa); + } + + [TestMethod] + [DataRow(false, MetadataPacketLayout.LttngCtf2)] + [DataRow(true, MetadataPacketLayout.LttngCtf2)] + [DataRow(false, MetadataPacketLayout.Ctf2Pmeta)] + [DataRow(true, MetadataPacketLayout.Ctf2Pmeta)] + public void VersionDetectingParserHandlesPacketizedCtf2Metadata(bool bigEndian, MetadataPacketLayout layout) + { + byte[] packets = MetadataPackets.Packetize(Encoding.UTF8.GetBytes(LTTngKernelMetadata), bigEndian, layout, maxContentBytesPerPacket: 512, paddingBytes: 4); + + var builder = new Ctf2TestMetadataBuilder(); + new CtfVersionDetectingMetadataParser(null, builder, true).Parse(new MemoryStream(packets)); + + Assert.AreEqual(2, builder.Streams.Count); + Assert.AreEqual(2, builder.AddedEvents.Count); + } + + [TestMethod] + public void RejectsUnsupportedVersion() + { + Assert.ThrowsException(() => Parse("\u001e{\"type\":\"preamble\",\"version\":3}")); + } + + [TestMethod] + public void ReadsBigEndianIntegers() + { + var builder = Parse(LTTngKernelMetadata); + var sched_switch = builder.AddedEvents.Single(e => e.Assignments["name"] == "sched_switch"); + var address = (CtfIntegerDescriptor)((ICtfStructDescriptor)sched_switch.TypeDeclarations["fields"]).GetField("_address_ipv4").TypeDescriptor; + + var value = (CtfIntegerValue)address.Read(new byte[] { 127, 0, 0, 1 }, 4); + Assert.AreEqual(0x7F000001ul, value.Value.ValueAsUlong); + } + + [TestMethod] + public void ReadsBigEndianFloatingPoint() + { + var builder = Parse(LTTngKernelMetadata); + var sched_switch = builder.AddedEvents.Single(e => e.Assignments["name"] == "sched_switch"); + var payload = (ICtfStructDescriptor)sched_switch.TypeDeclarations["fields"]; + + // 123.4f = 0x42F6CCCD + var bigEndian = (CtfFloatingPointDescriptor)payload.GetField("_ratio_be").TypeDescriptor; + Assert.AreEqual(123.4f, ((CtfFloatValue)bigEndian.Read(new byte[] { 0x42, 0xF6, 0xCC, 0xCD }, 4)).Value); + + // -2.0 = 0xC000000000000000 + var littleEndian = (CtfFloatingPointDescriptor)payload.GetField("_ratio").TypeDescriptor; + Assert.AreEqual(-2.0, ((CtfDoubleValue)littleEndian.Read(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0xC0 }, 8)).Value); + } + + [TestMethod] + public void ReadsVariableLengthIntegers() + { + var unsigned = new Ctf2VariableLengthIntegerDescriptor(false, 10, null); + var signed = new Ctf2VariableLengthIntegerDescriptor(true, 10, null); + + Assert.AreEqual(624485ul, ((CtfIntegerValue)unsigned.Read(new BytePacketReader(0xE5, 0x8E, 0x26))).Value.ValueAsUlong); + Assert.AreEqual(-123456L, ((CtfIntegerValue)signed.Read(new BytePacketReader(0xC0, 0xBB, 0x78))).Value.ValueAsLong); + } + + [TestMethod] + public void JsonReaderHandlesLargeNumbersAndEscapes() + { + var json = (Dictionary)Ctf2Json.Parse(@"{ ""a"": 18446744073709551615, ""b"": ""x\""\u0041\n"", ""c"": [true, false, null, -5] }"); + + Assert.AreEqual(ulong.MaxValue, ((Ctf2JsonNumber)json["a"]).AsULong()); + Assert.AreEqual("x\"A\n", json["b"]); + var array = (List)json["c"]; + Assert.AreEqual(true, array[0]); + Assert.AreEqual(false, array[1]); + Assert.IsNull(array[2]); + Assert.AreEqual(-5L, ((Ctf2JsonNumber)array[3]).AsLong()); + + Assert.ThrowsException(() => Ctf2Json.Parse("{\"a\": }")); + } + + // Payload members written the way LTTng-UST metadata (lttng-tools) writes them: every field location has an + // origin, and its path starts at the root of that scope. + private const string OriginRootedPayload = + @"[{""name"":""len"",""field-class"":""u8""}," + + @"{""name"":""values"",""field-class"":{""type"":""dynamic-length-array"",""element-field-class"":""u32"",""length-field-location"":{""origin"":""event-record-payload"",""path"":[""len""]}}}," + + @"{""name"":""nested"",""field-class"":{""type"":""structure"",""member-classes"":[{""name"":""n"",""field-class"":""u8""},{""name"":""text"",""field-class"":{""type"":""dynamic-length-string"",""length-field-location"":{""origin"":""event-record-payload"",""path"":[""nested"",""n""]}}}]}}," + + @"{""name"":""sel"",""field-class"":""u8""}," + + @"{""name"":""choice"",""field-class"":{""type"":""variant"",""selector-field-location"":{""origin"":""event-record-payload"",""path"":[""sel""]},""options"":[{""name"":""a"",""selector-field-ranges"":[[0,0]],""field-class"":""u8""},{""name"":""b"",""selector-field-ranges"":[[1,1]],""field-class"":""u32""}]}}," + + @"{""name"":""after"",""field-class"":""u32""}]"; + + [TestMethod] + public void ResolvesFieldLocationsRootedAtTheSameScope() + { + var builder = Parse(MetadataWithPayload(OriginRootedPayload)); + + // The event record header variant uses an "event-record-header" origin. + var header = builder.Streams.Single().EventHeader; + Assert.AreEqual("id", ((ICtfVariantDescriptor)header.GetField("v").TypeDescriptor).Switch); + Assert.IsInstanceOfType(header.GetField("id").TypeDescriptor, typeof(ICtfEnumDescriptor)); + + var payload = (ICtfStructDescriptor)builder.AddedEvents.Single().TypeDeclarations["fields"]; + Assert.AreEqual("_len", ((ICtfArrayDescriptor)payload.GetField("_values").TypeDescriptor).Index); + + var nested = (ICtfStructDescriptor)payload.GetField("_nested").TypeDescriptor; + Assert.AreEqual("_n", ((ICtfArrayDescriptor)nested.GetField("_text").TypeDescriptor).Index); + + Assert.AreEqual("_sel", ((ICtfVariantDescriptor)payload.GetField("_choice").TypeDescriptor).Switch); + Assert.IsInstanceOfType(payload.GetField("_sel").TypeDescriptor, typeof(ICtfEnumDescriptor)); + } + + [TestMethod] + public void DecodesFieldsLocatedFromTheSameScope() + { + var payload = (ICtfStructDescriptor)Parse(MetadataWithPayload(OriginRootedPayload)).AddedEvents.Single().TypeDeclarations["fields"]; + + var bytes = new List { 2 }; + bytes.AddRange(BitConverter.GetBytes(10u)); + bytes.AddRange(BitConverter.GetBytes(20u)); + bytes.Add(3); + bytes.AddRange(Encoding.UTF8.GetBytes("abc")); + bytes.Add(1); + bytes.AddRange(BitConverter.GetBytes(0xDEADBEEFu)); + bytes.AddRange(BitConverter.GetBytes(7u)); + var reader = new BufferPacketReader(bytes.ToArray()); + + var value = (CtfStructValue)payload.Read(reader); + + CollectionAssert.AreEqual(new uint[] { 10, 20 }, value.ReadFieldAsArray("_values").ReadAsUInt32Array()); + Assert.AreEqual("abc", ((CtfStructValue)value.FieldsByName["_nested"]).ReadFieldAsArray("_text").ReadAsString()); + var choice = (CtfVariantValue)value.FieldsByName["_choice"]; + Assert.AreEqual("_b", choice.Identifier); + Assert.AreEqual(0xDEADBEEFu, ((CtfIntegerValue)choice.Value).Value.ValueAsUlong); + Assert.AreEqual(7u, value.ReadFieldAsUInt32("_after")); + Assert.IsTrue(reader.EndOfStream); + } + + [TestMethod] + [DataRow(@"[{""name"":""values"",""field-class"":{""type"":""dynamic-length-array"",""element-field-class"":""u32"",""length-field-location"":{""origin"":""event-record-common-context"",""path"":[""ctx_len""]}}}]", "event-record-common-context")] + [DataRow(@"[{""name"":""len"",""field-class"":""u8""},{""name"":""nested"",""field-class"":{""type"":""structure"",""member-classes"":[{""name"":""values"",""field-class"":{""type"":""dynamic-length-array"",""element-field-class"":""u32"",""length-field-location"":{""origin"":""event-record-payload"",""path"":[""len""]}}}]}}]", "event-record-payload")] + [DataRow(@"[{""name"":""items"",""field-class"":{""type"":""static-length-array"",""length"":2,""element-field-class"":{""type"":""structure"",""member-classes"":[{""name"":""n"",""field-class"":""u8""},{""name"":""values"",""field-class"":{""type"":""dynamic-length-array"",""element-field-class"":""u32"",""length-field-location"":{""origin"":""event-record-payload"",""path"":[""items"",""n""]}}}]}}}]", "event-record-payload")] + [DataRow(@"[{""name"":""len"",""field-class"":""u8""},{""name"":""nested"",""field-class"":{""type"":""structure"",""member-classes"":[{""name"":""values"",""field-class"":{""type"":""dynamic-length-array"",""element-field-class"":""u32"",""length-field-location"":{""path"":[null,""len""]}}}]}}]", "null elements")] + public void RejectsUnsupportedFieldLocations(string payloadMembers, string expectedMessage) + { + var exception = Assert.ThrowsException(() => Parse(MetadataWithPayload(payloadMembers))); + StringAssert.Contains(exception.Message, expectedMessage); + } + + [TestMethod] + public void ParsesAndDecodesEventSpecificContext() + { + const string context = + @"[{""name"":""ctx_flag"",""field-class"":""u8""}," + + @"{""name"":""ctx_len"",""field-class"":""u8""}," + + @"{""name"":""ctx_values"",""field-class"":{""type"":""dynamic-length-array"",""element-field-class"":""u32"",""length-field-location"":{""origin"":""event-record-specific-context"",""path"":[""ctx_len""]}}}]"; + const string payload = @"[{""name"":""after"",""field-class"":""u32""}]"; + + var added = Parse(MetadataWithPayload(payload, context)).AddedEvents.Single(); + + var contextType = (ICtfStructDescriptor)added.TypeDeclarations["context"]; + CollectionAssert.AreEqual(new[] { "_ctx_flag", "_ctx_len", "_ctx_values" }, contextType.Fields.Select(f => f.Name).ToArray()); + Assert.AreEqual("_ctx_len", ((ICtfArrayDescriptor)contextType.GetField("_ctx_values").TypeDescriptor).Index); + + // An event's specific context is read right before its payload. + var bytes = new List { 9, 2 }; + bytes.AddRange(BitConverter.GetBytes(10u)); + bytes.AddRange(BitConverter.GetBytes(20u)); + bytes.AddRange(BitConverter.GetBytes(7u)); + var reader = new BufferPacketReader(bytes.ToArray()); + + var contextValue = (CtfStructValue)contextType.Read(reader); + var payloadValue = (CtfStructValue)((ICtfStructDescriptor)added.TypeDeclarations["fields"]).Read(reader); + + Assert.AreEqual(9u, contextValue.ReadFieldAsUInt32("_ctx_flag")); + CollectionAssert.AreEqual(new uint[] { 10, 20 }, contextValue.ReadFieldAsArray("_ctx_values").ReadAsUInt32Array()); + Assert.AreEqual(7u, payloadValue.ReadFieldAsUInt32("_after")); + Assert.IsTrue(reader.EndOfStream); + } + + private static string MetadataWithPayload(string payloadMembers, string specificContextMembers = null) + { + string specificContext = specificContextMembers == null + ? string.Empty + : @",""specific-context-field-class"":{""type"":""structure"",""member-classes"":" + specificContextMembers + "}"; + + return string.Concat(new[] + { + @"{""type"":""preamble"",""version"":2}", + @"{""type"":""field-class-alias"",""name"":""u8"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":8,""alignment"":8,""byte-order"":""little-endian""}}", + @"{""type"":""field-class-alias"",""name"":""u32"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":32,""alignment"":8,""byte-order"":""little-endian""}}", + @"{""type"":""trace-class"",""packet-header-field-class"":{""type"":""structure"",""member-classes"":[{""name"":""magic"",""field-class"":""u32""},{""name"":""stream_id"",""field-class"":""u32""}]}}", + @"{""type"":""data-stream-class"",""id"":0," + + @"""packet-context-field-class"":{""type"":""structure"",""member-classes"":[{""name"":""content_size"",""field-class"":""u32""}]}," + + @"""event-record-header-field-class"":{""type"":""structure"",""member-classes"":[{""name"":""id"",""field-class"":""u8""},{""name"":""v"",""field-class"":{""type"":""variant"",""selector-field-location"":{""origin"":""event-record-header"",""path"":[""id""]},""options"":[{""name"":""compact"",""selector-field-ranges"":[[0,254]],""field-class"":{""type"":""structure"",""member-classes"":[]}},{""name"":""extended"",""selector-field-ranges"":[[255,255]],""field-class"":{""type"":""structure"",""member-classes"":[{""name"":""id"",""field-class"":""u32""}]}}]}}]}," + + @"""event-record-common-context-field-class"":{""type"":""structure"",""member-classes"":[{""name"":""ctx_len"",""field-class"":""u32""}]}}", + @"{""type"":""event-record-class"",""data-stream-class-id"":0,""id"":0,""name"":""ust_event""" + specificContext + @",""payload-field-class"":{""type"":""structure"",""member-classes"":" + payloadMembers + "}}", + }.Select(fragment => "\u001e" + fragment + "\n")); + } + + private static Ctf2TestMetadataBuilder Parse(string metadata) + { + var builder = new Ctf2TestMetadataBuilder(); + new Ctf2MetadataParser(builder, prefixEventFieldNamesWithUnderscore: true).Parse(metadata); + return builder; + } + + private sealed class AddedEvent + { + public IReadOnlyDictionary Assignments { get; set; } + + public IReadOnlyDictionary TypeDeclarations { get; set; } + } + + private sealed class Ctf2TestMetadataBuilder + : ICtfMetadataBuilder + { + private readonly List clocks = new List(); + private readonly List streams = new List(); + + public List AddedEvents { get; } = new List(); + + public ICtfTraceDescriptor TraceDescriptor { get; private set; } + + public ICtfEnvironmentDescriptor EnvironmentDescriptor { get; private set; } + + public IReadOnlyList Clocks => this.clocks; + + public IReadOnlyDictionary ClocksByName => this.clocks.ToDictionary(c => c.Name); + + public IReadOnlyList Streams => this.streams; + + public IReadOnlyList Events => Array.Empty(); + + public void SetTraceDescriptor(ICtfTraceDescriptor traceDescriptor) => this.TraceDescriptor = traceDescriptor; + + public void SetEnvironmentDescriptor(ICtfEnvironmentDescriptor environmentDescriptor) => this.EnvironmentDescriptor = environmentDescriptor; + + public void AddEvent(IReadOnlyDictionary assignments, IReadOnlyDictionary typeDeclarations) + { + this.AddedEvents.Add(new AddedEvent { Assignments = assignments, TypeDeclarations = typeDeclarations }); + } + + public void AddClock(ICtfClockDescriptor clockDescriptor) => this.clocks.Add(clockDescriptor); + + public void AddStream(ICtfStreamDescriptor streamDescriptor) => this.streams.Add(streamDescriptor); + } + + private sealed class BufferPacketReader + : IPacketReader + { + private readonly byte[] bytes; + private int position; + + public BufferPacketReader(byte[] bytes) + { + this.bytes = bytes; + } + + public bool EndOfStream => this.position >= this.bytes.Length; + + public uint RemainingBufferedBitCount => (uint)(this.bytes.Length - this.position) * 8; + + public byte[] ReadBits(uint bitCount) + { + Assert.AreEqual(0u, bitCount % 8, "Only whole bytes are read in these tests."); + var result = new byte[bitCount / 8]; + Array.Copy(this.bytes, this.position, result, 0, result.Length); + this.position += result.Length; + return result; + } + + public byte[] ReadString() => throw new NotSupportedException(); + + public void Align(uint bitCount) + { + Assert.AreEqual(0u, (uint)(this.position * 8) % bitCount, "These tests only use byte-aligned fields."); + } + } + + private sealed class BytePacketReader + : IPacketReader + { + private readonly Queue bytes; + + public BytePacketReader(params byte[] bytes) + { + this.bytes = new Queue(bytes); + } + + public bool EndOfStream => this.bytes.Count == 0; + + public uint RemainingBufferedBitCount => (uint)this.bytes.Count * 8; + + public byte[] ReadBits(uint bitCount) + { + Assert.AreEqual(8u, bitCount); + return new[] { this.bytes.Dequeue() }; + } + + public byte[] ReadString() => throw new NotSupportedException(); + + public void Align(uint bitCount) + { + } + } + } +} diff --git a/CtfUnitTest/CtfMetadataTextTests.cs b/CtfUnitTest/CtfMetadataTextTests.cs new file mode 100644 index 0000000..947e4e3 --- /dev/null +++ b/CtfUnitTest/CtfMetadataTextTests.cs @@ -0,0 +1,113 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Buffers.Binary; +using System.IO; +using System.IO.Compression; +using System.Text; +using CtfPlayback.Metadata; +using CtfPlayback.Metadata.AntlrParser; +using CtfPlayback.Metadata.Interfaces; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace CtfUnitTest +{ + [TestClass] + public class CtfMetadataTextTests + { + // Contains multi-byte UTF-8 characters, which small packets split across packet boundaries. + private const string Text = + "\u001e{\"type\":\"preamble\",\"version\":2}\n" + + "\u001e{\"type\":\"trace-class\",\"environment\":{\"hostname\":\"h\u00f4te-\u00f1-\u65e5\u672c\"}}\n"; + + [TestMethod] + [DataRow(false, MetadataPacketLayout.Ctf18, 1)] + [DataRow(true, MetadataPacketLayout.Ctf18, 1)] + [DataRow(false, MetadataPacketLayout.LttngCtf2, 1)] + [DataRow(true, MetadataPacketLayout.LttngCtf2, 1)] + [DataRow(false, MetadataPacketLayout.LttngCtf2, 50)] + [DataRow(true, MetadataPacketLayout.LttngCtf2, int.MaxValue)] + [DataRow(false, MetadataPacketLayout.Ctf2Pmeta, 1)] + [DataRow(true, MetadataPacketLayout.Ctf2Pmeta, 1)] + [DataRow(false, MetadataPacketLayout.Ctf2Pmeta, 50)] + [DataRow(true, MetadataPacketLayout.Ctf2Pmeta, int.MaxValue)] + public void ReadsPacketizedMetadataInEitherByteOrder(bool bigEndian, MetadataPacketLayout layout, int maxContentBytesPerPacket) + { + byte[] packets = MetadataPackets.Packetize(Encoding.UTF8.GetBytes(Text), bigEndian, layout, maxContentBytesPerPacket, paddingBytes: 3); + + Assert.AreEqual(Text, CtfMetadataText.Read(packets)); + } + + [TestMethod] + public void ReturnsTextMetadataUnchanged() + { + Assert.AreEqual(Text, CtfMetadataText.Read(Encoding.UTF8.GetBytes(Text))); + + // Shorter than a packet header. + Assert.AreEqual("/* CTF 1.8 */", CtfMetadataText.Read(Encoding.UTF8.GetBytes("/* CTF 1.8 */"))); + } + + [TestMethod] + public void RejectsCorruptPackets() + { + byte[] content = Encoding.UTF8.GetBytes(Text); + + var badMagic = MetadataPackets.Packetize(content, true, MetadataPacketLayout.Ctf2Pmeta, maxContentBytesPerPacket: 20); + badMagic[44 + 20] ^= 0xFF; + Assert.ThrowsException(() => CtfMetadataText.Read(badMagic)); + + var badHeaderSize = MetadataPackets.Packetize(content, true, MetadataPacketLayout.Ctf2Pmeta); + BinaryPrimitives.WriteUInt32BigEndian(badHeaderSize.AsSpan(40), 37 * 8); + Assert.ThrowsException(() => CtfMetadataText.Read(badHeaderSize)); + + var headerLargerThanPacket = MetadataPackets.Packetize(content, false, MetadataPacketLayout.Ctf2Pmeta); + BinaryPrimitives.WriteUInt32LittleEndian(headerLargerThanPacket.AsSpan(40), 4096 * 8); + Assert.ThrowsException(() => CtfMetadataText.Read(headerLargerThanPacket)); + + var badContentSize = MetadataPackets.Packetize(content, false, MetadataPacketLayout.Ctf18); + BinaryPrimitives.WriteUInt32LittleEndian(badContentSize.AsSpan(24), (uint)((37 + content.Length) * 8) - 3); + Assert.ThrowsException(() => CtfMetadataText.Read(badContentSize)); + + // A big-endian header read as little endian would give an absurd content size. + var bigEndian = MetadataPackets.Packetize(content, true, MetadataPacketLayout.Ctf18); + Assert.AreEqual(Text, CtfMetadataText.Read(bigEndian)); + } + + [TestMethod] + public void ParsesBigEndianPacketizedCtf18Metadata() + { + // Real LTTng CTF 1.8 metadata, stored as little-endian packets. + byte[] original; + using (var trace = ZipFile.OpenRead(@"..\..\..\..\TestData\LTTng\lttng-kernel-trace.ctf")) + using (var metadata = trace.GetEntry("kernel/metadata").Open()) + { + original = CtfMetadataText.ReadAllBytes(metadata); + } + + Assert.AreEqual(MetadataPackets.Magic, BinaryPrimitives.ReadUInt32LittleEndian(original)); + + string text = CtfMetadataText.Read(original); + var reference = new TestCtfMetadataCustomization(); + new CtfAntlrMetadataParser(reference, reference).Parse(new MemoryStream(original)); + + var littleEndian = ParseWithVersionDetectingParser(original); + var bigEndian = ParseWithVersionDetectingParser(MetadataPackets.Packetize(Encoding.UTF8.GetBytes(text), true, MetadataPacketLayout.Ctf18, 4096, 5)); + + Assert.IsTrue(reference.Events.Count > 0); + foreach (var parsed in new[] { littleEndian, bigEndian }) + { + Assert.AreEqual(reference.Events.Count, parsed.Events.Count); + Assert.AreEqual(reference.Streams.Count, parsed.Streams.Count); + Assert.AreEqual(reference.Clocks.Count, parsed.Clocks.Count); + Assert.AreEqual(reference.TraceDescriptor.Uuid, parsed.TraceDescriptor.Uuid); + } + } + + private static ICtfMetadata ParseWithVersionDetectingParser(byte[] metadata) + { + var builder = new TestCtfMetadataCustomization(); + return new CtfVersionDetectingMetadataParser(builder, builder, false).Parse(new MemoryStream(metadata)); + } + } +} diff --git a/CtfUnitTest/FloatingPointTest.cs b/CtfUnitTest/FloatingPointTest.cs index f7cce64..169a419 100644 --- a/CtfUnitTest/FloatingPointTest.cs +++ b/CtfUnitTest/FloatingPointTest.cs @@ -3,6 +3,8 @@ using System; using System.Collections.Generic; +using CtfPlayback.FieldValues; +using CtfPlayback.Metadata.InternalHelpers; using CtfPlayback.Metadata.Types; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -26,7 +28,7 @@ public void FloatingPointTests32() { new byte[] { 0x00, 0x00, 0xC8, 0x3C }, 0.0244140625f}, // From https://en.wikipedia.org/wiki/Single-precision_floating-point_format - // CTF is supposed to follow IEEE 754-2008 format - https://diamon.org/ctf/#spec4.1.7 + // CTF is supposed to follow IEEE 754-2008 format - https://diamon.org/ctf/v1.8.3/#spec4.1.7 { new byte[] { 0x00, 0x00, 0x20, 0x3E }, 0.15625f }, { new byte[] { 0x00, 0x20, 0xA7, 0x44 }, 1337.0f }, { new byte[] { 0x00, 0x00, 0x46, 0x41 }, 12.375f }, @@ -90,5 +92,49 @@ public void FloatingPointTests64() Assert.AreEqual(rawDoubleDict.Value, fpDouble); } } + + [TestMethod] + [TestCategory("UnitTest")] + [DataRow(null)] + [DataRow("le")] + [DataRow("native")] + [DataRow("be")] + [DataRow("network")] + public void FloatingPointByteOrder(string byteOrder) + { + bool bigEndian = byteOrder == "be" || byteOrder == "network"; + + // 123.4f = 0x42F6CCCD, -2.0 = 0xC000000000000000 + byte[] float32 = { 0xCD, 0xCC, 0xF6, 0x42 }; + byte[] float64 = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC0 }; + if (bigEndian) + { + Array.Reverse(float32); + Array.Reverse(float64); + } + + var single = (CtfFloatValue)CreateDescriptor(8, 24, byteOrder).Read(float32, float32.Length); + Assert.AreEqual(123.4f, single.Value); + + var dbl = (CtfDoubleValue)CreateDescriptor(11, 53, byteOrder).Read(float64, float64.Length); + Assert.AreEqual(-2.0, dbl.Value); + + // The caller's buffer must not be modified. + Assert.AreEqual(bigEndian ? (byte)0x42 : (byte)0xCD, float32[0]); + } + + private static CtfFloatingPointDescriptor CreateDescriptor(int exponent, int mantissa, string byteOrder) + { + var bag = new CtfPropertyBag(); + bag.AddValue("exp_dig", exponent.ToString()); + bag.AddValue("mant_dig", mantissa.ToString()); + bag.AddValue("align", "8"); + if (byteOrder != null) + { + bag.AddValue("byte_order", byteOrder); + } + + return new CtfFloatingPointDescriptor(bag); + } } } \ No newline at end of file diff --git a/CtfUnitTest/IntegerDescriptorTests.cs b/CtfUnitTest/IntegerDescriptorTests.cs new file mode 100644 index 0000000..a5a50d9 --- /dev/null +++ b/CtfUnitTest/IntegerDescriptorTests.cs @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Linq; +using CtfPlayback.FieldValues; +using CtfPlayback.Metadata.InternalHelpers; +using CtfPlayback.Metadata.Types; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace CtfUnitTest +{ + [TestClass] + public class IntegerDescriptorTests + { + [TestMethod] + [DataRow("le")] + [DataRow("be")] + public void ReadsSignedIntegersOfEverySize(string byteOrder) + { + foreach (int size in new[] { 8, 16, 24, 32, 40, 48, 56, 64 }) + { + long max = size == 64 ? long.MaxValue : (1L << (size - 1)) - 1; + long min = size == 64 ? long.MinValue : -(1L << (size - 1)); + var values = new List { 0, 1, -1, max, min, max / 3, min / 3 }; + if (size > 32) + { + // Values with bits at and above bit 31 set, which a 32-bit sign mask mishandles. + values.AddRange(new[] { 1L << 31, 1L << 32, -(1L << 32), 0x7FFFFFFFL, 0x80000000L, -0x80000001L }); + } + + foreach (long value in values) + { + Assert.AreEqual(value, ReadSigned(size, byteOrder, value), $"size={size} byteOrder={byteOrder} value=0x{value:X}"); + } + } + } + + [TestMethod] + public void ReadsSigned64BitBigEndian() + { + var descriptor = CreateDescriptor(64, signed: true, "be"); + + var value = (CtfIntegerValue)descriptor.Read(new byte[] { 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00 }, 8); + Assert.AreEqual(0x0000000100000000L, value.Value.ValueAsLong); + + value = (CtfIntegerValue)descriptor.Read(new byte[] { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE }, 8); + Assert.AreEqual(-2L, value.Value.ValueAsLong); + + value = (CtfIntegerValue)descriptor.Read(new byte[] { 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 8); + Assert.IsTrue(value.Value.TryGetInt64(out long minimum)); + Assert.AreEqual(long.MinValue, minimum); + } + + [TestMethod] + [DataRow("le")] + [DataRow("be")] + public void ReadsUnsigned64Bit(string byteOrder) + { + foreach (ulong value in new[] { 0ul, 1ul, 0x80000000ul, 0x100000000ul, ulong.MaxValue }) + { + byte[] bytes = BitConverter.GetBytes(value); + if (byteOrder == "be") + { + Array.Reverse(bytes); + } + + var result = (CtfIntegerValue)CreateDescriptor(64, signed: false, byteOrder).Read(bytes, bytes.Length); + Assert.AreEqual(value, result.Value.ValueAsUlong); + } + } + + private static long ReadSigned(int size, string byteOrder, long value) + { + // Two's complement, truncated to the integer's size. + byte[] bytes = BitConverter.GetBytes(value).Take(size / 8).ToArray(); + if (byteOrder == "be") + { + Array.Reverse(bytes); + } + + var result = (CtfIntegerValue)CreateDescriptor(size, signed: true, byteOrder).Read(bytes, bytes.Length); + return result.Value.ValueAsLong; + } + + private static CtfIntegerDescriptor CreateDescriptor(int size, bool signed, string byteOrder) + { + var bag = new CtfPropertyBag(); + bag.AddValue("size", size.ToString()); + bag.AddValue("align", "8"); + bag.AddValue("signed", signed ? "true" : "false"); + bag.AddValue("byte_order", byteOrder); + return new CtfIntegerDescriptor(bag); + } + } +} diff --git a/CtfUnitTest/MetadataPackets.cs b/CtfUnitTest/MetadataPackets.cs new file mode 100644 index 0000000..e6c9fe2 --- /dev/null +++ b/CtfUnitTest/MetadataPackets.cs @@ -0,0 +1,87 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Buffers.Binary; +using System.IO; + +namespace CtfUnitTest +{ + /// + /// Metadata packet header layouts. + /// + public enum MetadataPacketLayout + { + /// CTF 1.8 section 7.1: 37 bytes, version 1.8. + Ctf18, + + /// What LTTng 2.15+ writes for CTF 2 metadata: the 37-byte CTF 1.8 header with version 2.0. + LttngCtf2, + + /// CTF2-PMETA-1.0: 44 bytes (3 reserved bytes and a header size field added), version 2.0. + Ctf2Pmeta, + } + + /// + /// Builds packetized metadata streams. + /// + internal static class MetadataPackets + { + internal const uint Magic = 0x75d11d57; + + /// Metadata content + /// Byte order of the header's integer fields + /// Packet header layout + /// Content is split into packets of at most this many bytes + /// Padding after the content of each packet + internal static byte[] Packetize(byte[] content, bool bigEndian, MetadataPacketLayout layout, int maxContentBytesPerPacket = int.MaxValue, int paddingBytes = 0) + { + int headerSize = layout == MetadataPacketLayout.Ctf2Pmeta ? 44 : 37; + byte majorVersion = layout == MetadataPacketLayout.Ctf18 ? (byte)1 : (byte)2; + byte minorVersion = layout == MetadataPacketLayout.Ctf18 ? (byte)8 : (byte)0; + + var stream = new MemoryStream(); + int offset = 0; + do + { + int chunk = Math.Min(maxContentBytesPerPacket, content.Length - offset); + uint contentBits = (uint)((headerSize + chunk) * 8); + uint packetBits = (uint)((headerSize + chunk + paddingBytes) * 8); + + WriteUInt32(stream, Magic, bigEndian); + stream.Write(new byte[16], 0, 16); + WriteUInt32(stream, 0, bigEndian); + WriteUInt32(stream, contentBits, bigEndian); + WriteUInt32(stream, packetBits, bigEndian); + stream.Write(new byte[] { 0, 0, 0, majorVersion, minorVersion }, 0, 5); + if (layout == MetadataPacketLayout.Ctf2Pmeta) + { + stream.Write(new byte[3], 0, 3); + WriteUInt32(stream, (uint)(headerSize * 8), bigEndian); + } + + stream.Write(content, offset, chunk); + stream.Write(new byte[paddingBytes], 0, paddingBytes); + offset += chunk; + } + while (offset < content.Length); + + return stream.ToArray(); + } + + internal static void WriteUInt32(Stream stream, uint value, bool bigEndian) + { + var bytes = new byte[4]; + if (bigEndian) + { + BinaryPrimitives.WriteUInt32BigEndian(bytes, value); + } + else + { + BinaryPrimitives.WriteUInt32LittleEndian(bytes, value); + } + + stream.Write(bytes, 0, bytes.Length); + } + } +} diff --git a/LTTngCds/CookerData/LTTngEvent.cs b/LTTngCds/CookerData/LTTngEvent.cs index cb8f5ad..d253e84 100644 --- a/LTTngCds/CookerData/LTTngEvent.cs +++ b/LTTngCds/CookerData/LTTngEvent.cs @@ -50,10 +50,21 @@ internal LTTngEvent(LTTngEvent lttngEvent, long timestampAdjustmentNano) : this( } /// - /// Event Id + /// Event Id. Event ids are only unique within a stream, see . /// public uint Id => this.eventDescriptor.Id; + /// + /// Id of the stream (LTTng channel) that contains the event. + /// + public uint StreamId => this.ctfEvent.StreamId; + + /// + /// Metadata describing the event's class. A single instance is shared by all events of that class within a + /// trace, so it identifies the event class even across traces whose stream and event ids overlap. + /// + public IEventDescriptor EventDescriptor => this.eventDescriptor; + /// /// Event name /// @@ -84,6 +95,12 @@ internal LTTngEvent(LTTngEvent lttngEvent, long timestampAdjustmentNano) : this( /// public CtfStructValue StreamDefinedEventContext => this.ctfEvent.StreamDefinedEventContext as CtfStructValue; + /// + /// Context specific to the event's class (a CTF 2 event record specific context), read after the stream + /// defined event context and before the payload. Null when the event class doesn't define one. + /// + public CtfStructValue SpecificContext => this.ctfEvent.Context as CtfStructValue; + /// /// Event payload /// diff --git a/LTTngCds/CtfExtensions/Descriptors/EventDescriptor.cs b/LTTngCds/CtfExtensions/Descriptors/EventDescriptor.cs index 4c7d37f..d7f8923 100644 --- a/LTTngCds/CtfExtensions/Descriptors/EventDescriptor.cs +++ b/LTTngCds/CtfExtensions/Descriptors/EventDescriptor.cs @@ -31,6 +31,11 @@ internal EventDescriptor( this.Name = string.Intern(name.Trim(new[] { '"' })); this.SetPayload(typeDeclarations); + + if (typeDeclarations.TryGetValue("context", out var context)) + { + this.Context = context; + } } public uint Id { get; private set; } @@ -41,9 +46,9 @@ internal EventDescriptor( public uint LogLevel { get; private set; } - // LTTng defines no context for an event + // CTF 1.8 LTTng traces define no context for an event; CTF 2 traces may define a specific context. - public ICtfTypeDescriptor Context => null; + public ICtfTypeDescriptor Context { get; } public ICtfTypeDescriptor Payload { get; private set; } diff --git a/LTTngCds/CtfExtensions/FolderInput/LTTngFolderInput.cs b/LTTngCds/CtfExtensions/FolderInput/LTTngFolderInput.cs index 5e7ddbb..576d920 100644 --- a/LTTngCds/CtfExtensions/FolderInput/LTTngFolderInput.cs +++ b/LTTngCds/CtfExtensions/FolderInput/LTTngFolderInput.cs @@ -35,7 +35,7 @@ public LTTngFolderInput(string folderPath) Debug.Assert(traceDirectoryPath != null, nameof(traceDirectoryPath) + " != null"); var associatedEntries = Directory.GetFiles(traceDirectoryPath).Where(entry => - Path.GetFileName(entry).StartsWith("chan")); + LTTngStreamFiles.IsEventStreamFile(Path.GetFileName(entry))); traceInput.EventStreams = associatedEntries.Select( fileName => new LTTngFileInputStream(fileName)).Cast().ToList(); diff --git a/LTTngCds/CtfExtensions/LTTngMetadata.cs b/LTTngCds/CtfExtensions/LTTngMetadata.cs index 49ffa4d..78bd29b 100644 --- a/LTTngCds/CtfExtensions/LTTngMetadata.cs +++ b/LTTngCds/CtfExtensions/LTTngMetadata.cs @@ -15,7 +15,7 @@ internal class LTTngMetadata private readonly List streams = new List(); private readonly List clocks = new List(); private readonly List events = new List(); - private readonly Dictionary eventsById = new Dictionary(); + private readonly Dictionary<(int StreamId, uint EventId), ICtfEventDescriptor> eventsByStreamAndId = new Dictionary<(int StreamId, uint EventId), ICtfEventDescriptor>(); private readonly Dictionary clocksByName = new Dictionary(); public ICtfTraceDescriptor TraceDescriptor { get; private set; } @@ -30,7 +30,13 @@ internal class LTTngMetadata public IReadOnlyList Events => this.events; - public IReadOnlyDictionary EventByEventId => this.eventsById; + /// + /// Event ids are only unique within a stream (e.g. with multiple channels, or in CTF 2 traces). + /// + public bool TryGetEvent(uint streamId, uint eventId, out ICtfEventDescriptor eventDescriptor) + { + return this.eventsByStreamAndId.TryGetValue(((int)streamId, eventId), out eventDescriptor); + } public void SetTraceDescriptor(ICtfTraceDescriptor traceDescriptor) { @@ -52,7 +58,7 @@ public void AddEvent( { var eventDescriptor = new EventDescriptor(assignments, typeDeclarations); this.events.Add(eventDescriptor); - this.eventsById.Add(eventDescriptor.Id, eventDescriptor); + this.eventsByStreamAndId.Add((eventDescriptor.Stream, eventDescriptor.Id), eventDescriptor); } public void AddClock(ICtfClockDescriptor clockDescriptor) diff --git a/LTTngCds/CtfExtensions/LTTngPlaybackCustomization.cs b/LTTngCds/CtfExtensions/LTTngPlaybackCustomization.cs index 0e50e8b..2628802 100644 --- a/LTTngCds/CtfExtensions/LTTngPlaybackCustomization.cs +++ b/LTTngCds/CtfExtensions/LTTngPlaybackCustomization.cs @@ -10,7 +10,6 @@ using CtfPlayback.FieldValues; using CtfPlayback.Inputs; using CtfPlayback.Metadata; -using CtfPlayback.Metadata.AntlrParser; using CtfPlayback.Metadata.Interfaces; using LTTngCds.CookerData; using LTTngCds.CtfExtensions.Descriptors; @@ -41,7 +40,10 @@ public virtual ICtfMetadataParser CreateMetadataParser( ICtfTraceInput traceInput) { this.metadataCustomization.PrepareForNewTrace(traceInput.MetadataStream); - return new CtfAntlrMetadataParser(this.metadataCustomization, this.metadataCustomization.LTTngMetadata); + return new CtfVersionDetectingMetadataParser( + this.metadataCustomization, + this.metadataCustomization.LTTngMetadata, + prefixCtf2EventFieldNamesWithUnderscore: true); } public bool GetTimestampsFromPacketContext( @@ -223,22 +225,24 @@ public ulong GetPacketContentBitCount(ICtfPacket ctfPacket) public ICtfEventDescriptor GetEventDescriptor(ICtfEvent ctfEvent, ICtfMetadata metadata) { uint id = this.GetEventId(ctfEvent); + uint streamId = ctfEvent.StreamId; ICtfEventDescriptor eventDescriptor; if (metadata is LTTngMetadata typedMetadata) { // optimization if we got our own typed metadata back // we check for success below - typedMetadata.EventByEventId.TryGetValue(id, out eventDescriptor); + typedMetadata.TryGetEvent(streamId, id, out eventDescriptor); } else { - eventDescriptor = metadata.Events.FirstOrDefault(x => x.Id == id); + eventDescriptor = metadata.Events.FirstOrDefault(x => + x.Id == id && (!(x is EventDescriptor lttngDescriptor) || lttngDescriptor.Stream == streamId)); } if (eventDescriptor == null) { - throw new LTTngPlaybackException($"Unable to find event descriptor for event id={id}."); + throw new LTTngPlaybackException($"Unable to find event descriptor for stream id={streamId}, event id={id}."); } return eventDescriptor; diff --git a/LTTngCds/CtfExtensions/LTTngStreamFiles.cs b/LTTngCds/CtfExtensions/LTTngStreamFiles.cs new file mode 100644 index 0000000..80efbb2 --- /dev/null +++ b/LTTngCds/CtfExtensions/LTTngStreamFiles.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; + +namespace LTTngCds.CtfExtensions +{ + internal static class LTTngStreamFiles + { + /// + /// LTTng names event stream files "<channel name>_<cpu>", e.g. "channel0_3" or "sw-chan_3". + /// + /// File name without directory + /// true if the file is an event stream + internal static bool IsEventStreamFile(string fileName) + { + if (string.IsNullOrEmpty(fileName) || + fileName.StartsWith(".", StringComparison.Ordinal) || + StringComparer.Ordinal.Equals(fileName, "metadata")) + { + return false; + } + + int separatorIndex = fileName.LastIndexOf('_'); + if (separatorIndex <= 0 || separatorIndex == fileName.Length - 1) + { + return false; + } + + for (int index = separatorIndex + 1; index < fileName.Length; index++) + { + if (!char.IsDigit(fileName[index])) + { + return false; + } + } + + return true; + } + } +} diff --git a/LTTngCds/CtfExtensions/ZipArchiveInput/LTTngZipArchiveInput.cs b/LTTngCds/CtfExtensions/ZipArchiveInput/LTTngZipArchiveInput.cs index b843252..c017780 100644 --- a/LTTngCds/CtfExtensions/ZipArchiveInput/LTTngZipArchiveInput.cs +++ b/LTTngCds/CtfExtensions/ZipArchiveInput/LTTngZipArchiveInput.cs @@ -39,7 +39,7 @@ public LTTngZipArchiveInput(ZipArchive archive) var associatedArchiveEntries = archive.Entries.Where(entry => Path.GetDirectoryName(entry.FullName) == traceDirectoryPath && - Path.GetFileName(entry.FullName).StartsWith("chan")); + LTTngStreamFiles.IsEventStreamFile(Path.GetFileName(entry.FullName))); traceInput.EventStreams = associatedArchiveEntries.Select( archiveEntry => new LTTngZipArchiveInputStream(archiveEntry)).Cast().ToList(); diff --git a/LTTngDataExtUnitTest/Ctf2TestTraces.cs b/LTTngDataExtUnitTest/Ctf2TestTraces.cs new file mode 100644 index 0000000..1e0150e --- /dev/null +++ b/LTTngDataExtUnitTest/Ctf2TestTraces.cs @@ -0,0 +1,160 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text; +using LTTngDataExtensions.DataOutputTypes; +using LTTngDataExtensions.SourceDataCookers; +using Microsoft.Performance.SDK.Extensibility; +using Microsoft.Performance.SDK.Processing; +using Microsoft.Performance.Toolkit.Engine; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace LTTngDataExtUnitTest +{ + /// + /// Builds small synthetic CTF 2 traces with one data stream class (id 0), and processes them. + /// + internal static class Ctf2TestTraces + { + internal const string StreamFileName = "channel0_0"; + + /// + /// Metadata with field class aliases u8, u16, u32, s32 and u64, and the given event record classes. + /// + internal static string Metadata(string domain, params string[] eventRecordClasses) + { + var fragments = new List + { + "{\"type\":\"preamble\",\"version\":2}", + "{\"type\":\"field-class-alias\",\"name\":\"u64-ts\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":64,\"alignment\":8,\"byte-order\":\"little-endian\",\"roles\":[\"default-clock-timestamp\"]}}", + Integer("u8", 8, false), + Integer("u16", 16, false), + Integer("u32", 32, false), + Integer("s32", 32, true), + Integer("u64", 64, false), + "{\"type\":\"field-class-alias\",\"name\":\"er-header-compact\",\"field-class\":{\"type\":\"structure\",\"minimum-alignment\":8,\"member-classes\":[{\"name\":\"id\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":5,\"byte-order\":\"little-endian\"}},{\"name\":\"v\",\"field-class\":{\"type\":\"variant\",\"selector-field-location\":{\"path\":[\"id\"]},\"options\":[{\"name\":\"compact\",\"selector-field-ranges\":[[0,30]],\"field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"timestamp\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":27,\"byte-order\":\"little-endian\",\"roles\":[\"default-clock-timestamp\"]}}]}},{\"name\":\"extended\",\"selector-field-ranges\":[[31,31]],\"field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"id\",\"field-class\":\"u32\"},{\"name\":\"timestamp\",\"field-class\":\"u64-ts\"}]}}]}}]}}", + "{\"type\":\"trace-class\",\"environment\":{\"domain\":\"" + domain + "\"},\"packet-header-field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"magic\",\"field-class\":\"u32\"},{\"name\":\"uuid\",\"field-class\":{\"type\":\"static-length-blob\",\"length\":16}},{\"name\":\"stream_id\",\"field-class\":\"u32\"}]}}", + "{\"type\":\"clock-class\",\"id\":\"monotonic\",\"name\":\"monotonic\",\"frequency\":1000000000,\"origin\":\"unix-epoch\",\"offset-from-origin\":{\"seconds\":1790000000}}", + "{\"type\":\"data-stream-class\",\"id\":0,\"default-clock-class-id\":\"monotonic\",\"event-record-header-field-class\":\"er-header-compact\",\"packet-context-field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"timestamp_begin\",\"field-class\":\"u64-ts\"},{\"name\":\"timestamp_end\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":64,\"alignment\":8,\"byte-order\":\"little-endian\",\"roles\":[\"packet-end-default-clock-timestamp\"]}},{\"name\":\"content_size\",\"field-class\":\"u64\"},{\"name\":\"packet_size\",\"field-class\":\"u64\"},{\"name\":\"events_discarded\",\"field-class\":\"u64\"},{\"name\":\"cpu_id\",\"field-class\":\"u32\"}]}}", + }; + + fragments.AddRange(eventRecordClasses); + return string.Concat(fragments.Select(fragment => "\u001e" + fragment + "\n")); + } + + /// + /// An event record class of data stream class 0. Members are JSON arrays of structure member classes. + /// + internal static string EventRecordClass(uint id, string name, string payloadMembers, string specificContextMembers = null) + { + string specificContext = specificContextMembers == null + ? string.Empty + : ",\"specific-context-field-class\":{\"type\":\"structure\",\"member-classes\":" + specificContextMembers + "}"; + + return "{\"type\":\"event-record-class\",\"data-stream-class-id\":0,\"id\":" + id + ",\"name\":\"" + name + "\"" + + specificContext + + ",\"payload-field-class\":{\"type\":\"structure\",\"member-classes\":" + payloadMembers + "}}"; + } + + /// + /// An event with a compact header (5-bit event record class id, then a 27-bit timestamp), followed by the + /// given context and payload bytes. + /// + internal static byte[] Event(uint id, uint timestamp, params byte[][] fields) + { + Assert.IsTrue(id < 31 && timestamp < (1u << 27)); + return BitConverter.GetBytes(id | (timestamp << 5)).Concat(fields.SelectMany(field => field)).ToArray(); + } + + /// + /// A packet of data stream 0 with the given events. + /// + internal static byte[] Packet(uint timestampBegin, uint timestampEnd, params byte[][] events) + { + const int headerAndContextBytes = 4 + 16 + 4 + (5 * 8) + 4; + ulong sizeInBits = (ulong)(headerAndContextBytes + events.Sum(e => e.Length)) * 8; + + var stream = new MemoryStream(); + var writer = new BinaryWriter(stream); + writer.Write(0xC1FC1FC1u); + writer.Write(new byte[16]); + writer.Write(0u); + writer.Write((ulong)timestampBegin); + writer.Write((ulong)timestampEnd); + writer.Write(sizeInBits); + writer.Write(sizeInBits); + writer.Write(0ul); + writer.Write(0u); + foreach (var e in events) + { + writer.Write(e); + } + + return stream.ToArray(); + } + + /// + /// Zips the traces (each a metadata file and one stream file in its directory), processes the archive, and + /// returns the generic events. + /// + internal static List ProcessGenericEvents(params (string Directory, string Metadata, byte[] Stream)[] traces) + { + string path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".ctf"); + try + { + using (var archive = ZipFile.Open(path, ZipArchiveMode.Create)) + { + foreach (var trace in traces) + { + WriteEntry(archive, trace.Directory + "/metadata", Encoding.UTF8.GetBytes(trace.Metadata)); + WriteEntry(archive, trace.Directory + "/" + StreamFileName, trace.Stream); + } + } + + using (var runtime = Engine.Create(new FileDataSource(path))) + { + var cooker = new LTTngGenericEventDataCooker(); + runtime.EnableCooker(cooker.Path); + var results = runtime.Process(); + + Assert.AreEqual(0, results.ProcessingErrors.Count(), string.Join(Environment.NewLine, results.ProcessingErrors.Select(e => e.ProcessFault?.ToString()))); + + var events = results.QueryOutput>( + new DataOutputPath(cooker.Path, nameof(LTTngGenericEventDataCooker.Events))); + + return Enumerable.Range(0, (int)events.Count).Select(i => events[i]).ToList(); + } + } + finally + { + // The processor may still hold the archive open after the run. + try + { + File.Delete(path); + } + catch (IOException) + { + } + } + } + + private static string Integer(string alias, int length, bool signed) + { + return "{\"type\":\"field-class-alias\",\"name\":\"" + alias + "\",\"field-class\":{\"type\":\"fixed-length-" + (signed ? "signed" : "unsigned") + + "-integer\",\"length\":" + length + ",\"alignment\":8,\"byte-order\":\"little-endian\"}}"; + } + + private static void WriteEntry(ZipArchive archive, string name, byte[] content) + { + using (var stream = archive.CreateEntry(name).Open()) + { + stream.Write(content, 0, content.Length); + } + } + } +} diff --git a/LTTngDataExtUnitTest/LTTngCtf2EventContextTests.cs b/LTTngDataExtUnitTest/LTTngCtf2EventContextTests.cs new file mode 100644 index 0000000..446385a --- /dev/null +++ b/LTTngDataExtUnitTest/LTTngCtf2EventContextTests.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Linq; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace LTTngDataExtUnitTest +{ + /// + /// A CTF 2 event record class can define a specific context, which is recorded between the stream's common + /// context and the payload. + /// + [TestClass] + public class LTTngCtf2EventContextTests + { + // Context members of different sizes, so a context that isn't read, or is read with the wrong size, shifts + // every payload field. + private const string ContextMembers = @"[{""name"":""ctx_flag"",""field-class"":""u8""},{""name"":""ctx_id"",""field-class"":""u32""}]"; + private const string PayloadMembers = @"[{""name"":""value"",""field-class"":""s32""},{""name"":""count"",""field-class"":""u16""}]"; + + [TestMethod] + public void DecodesSpecificContextAndFollowingPayload() + { + // Event record class 0 has a specific context; class 1 in the same stream doesn't. + string metadata = Ctf2TestTraces.Metadata( + "ust", + Ctf2TestTraces.EventRecordClass(0, "with_context", PayloadMembers, ContextMembers), + Ctf2TestTraces.EventRecordClass(1, "without_context", PayloadMembers)); + + byte[] stream = Ctf2TestTraces.Packet( + 1000, + 1200, + Ctf2TestTraces.Event(0, 1000, Context(7, 0x11223344), Payload(-5, 42)), + Ctf2TestTraces.Event(1, 1100, Payload(-6, 43)), + Ctf2TestTraces.Event(0, 1200, Context(8, 0x55667788), Payload(-7, 44))); + + var events = Ctf2TestTraces.ProcessGenericEvents(("trace/ust/uid/1000/64-bit", metadata, stream)) + .Select(e => $"{e.EventName}: " + string.Join(", ", e.FieldNames.Zip(e.FieldValues, (name, value) => $"{name} = {value}"))) + .ToList(); + + CollectionAssert.AreEqual( + new[] + { + "with_context: _ctx_flag = 7, _ctx_id = 287454020, _value = -5, _count = 42", + "without_context: _value = -6, _count = 43", + "with_context: _ctx_flag = 8, _ctx_id = 1432778632, _value = -7, _count = 44", + }, + events, + string.Join(Environment.NewLine, events)); + } + + private static byte[] Context(byte flag, uint id) + { + return new[] { flag }.Concat(BitConverter.GetBytes(id)).ToArray(); + } + + private static byte[] Payload(int value, ushort count) + { + return BitConverter.GetBytes(value).Concat(BitConverter.GetBytes(count)).ToArray(); + } + } +} diff --git a/LTTngDataExtUnitTest/LTTngCtf2UnitTest.cs b/LTTngDataExtUnitTest/LTTngCtf2UnitTest.cs new file mode 100644 index 0000000..100cb5a --- /dev/null +++ b/LTTngDataExtUnitTest/LTTngCtf2UnitTest.cs @@ -0,0 +1,221 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text; +using LTTngDataExtensions.DataOutputTypes; +using LTTngDataExtensions.SourceDataCookers; +using LTTngDataExtensions.SourceDataCookers.Thread; +using LTTngDataExtensions.Tables; +using Microsoft.Performance.SDK.Extensibility; +using Microsoft.Performance.SDK.Processing; +using Microsoft.Performance.Toolkit.Engine; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace LTTngDataExtUnitTest +{ + /// + /// Processes a small synthetic LTTng kernel trace written in the CTF 2 format, with channels that are not named + /// "chan*" and event ids that are only unique within a channel (as written by LTTng 2.15+). + /// + [TestClass] + public class LTTngCtf2UnitTest + { + private const string Metadata = + "\u001e{\"type\":\"preamble\",\"version\":2}\n" + + "\u001e{\"type\":\"field-class-alias\",\"name\":\"u64-ts\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":64,\"alignment\":8,\"byte-order\":\"little-endian\",\"roles\":[\"default-clock-timestamp\"]}}\n" + + "\u001e{\"type\":\"field-class-alias\",\"name\":\"u64\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":64,\"alignment\":8,\"byte-order\":\"little-endian\"}}\n" + + "\u001e{\"type\":\"field-class-alias\",\"name\":\"u32\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":32,\"alignment\":8,\"byte-order\":\"little-endian\"}}\n" + + "\u001e{\"type\":\"field-class-alias\",\"name\":\"s32\",\"field-class\":{\"type\":\"fixed-length-signed-integer\",\"length\":32,\"alignment\":8,\"byte-order\":\"little-endian\"}}\n" + + "\u001e{\"type\":\"field-class-alias\",\"name\":\"comm\",\"field-class\":{\"type\":\"static-length-string\",\"length\":16}}\n" + + "\u001e{\"type\":\"field-class-alias\",\"name\":\"er-header-compact\",\"field-class\":{\"type\":\"structure\",\"minimum-alignment\":8,\"member-classes\":[{\"name\":\"id\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":5,\"byte-order\":\"little-endian\",\"mappings\":{\"compact\":[[0,30]],\"extended\":[[31,31]]}}},{\"name\":\"v\",\"field-class\":{\"type\":\"variant\",\"selector-field-location\":{\"path\":[\"id\"]},\"options\":[{\"name\":\"compact\",\"selector-field-ranges\":[[0,30]],\"field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"timestamp\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":27,\"byte-order\":\"little-endian\",\"roles\":[\"default-clock-timestamp\"]}}]}},{\"name\":\"extended\",\"selector-field-ranges\":[[31,31]],\"field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"id\",\"field-class\":\"u32\"},{\"name\":\"timestamp\",\"field-class\":\"u64-ts\"}]}}]}}]}}\n" + + "\u001e{\"type\":\"field-class-alias\",\"name\":\"pkt-ctx\",\"field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"timestamp_begin\",\"field-class\":\"u64-ts\"},{\"name\":\"timestamp_end\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":64,\"alignment\":8,\"byte-order\":\"little-endian\",\"roles\":[\"packet-end-default-clock-timestamp\"]}},{\"name\":\"content_size\",\"field-class\":\"u64\"},{\"name\":\"packet_size\",\"field-class\":\"u64\"},{\"name\":\"events_discarded\",\"field-class\":\"u64\"},{\"name\":\"cpu_id\",\"field-class\":\"u32\"}]}}\n" + + "\u001e{\"type\":\"field-class-alias\",\"name\":\"pkt-header\",\"field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"magic\",\"field-class\":\"u32\"},{\"name\":\"uuid\",\"field-class\":{\"type\":\"static-length-blob\",\"length\":16}},{\"name\":\"stream_id\",\"field-class\":\"u32\"}]}}\n" + + "\u001e{\"type\":\"trace-class\",\"environment\":{\"domain\":\"kernel\",\"tracer_name\":\"lttng-modules\",\"tracer_major\":2,\"tracer_minor\":16},\"packet-header-field-class\":\"pkt-header\"}\n" + + "\u001e{\"type\":\"clock-class\",\"id\":\"monotonic\",\"name\":\"monotonic\",\"frequency\":1000000000,\"origin\":\"unix-epoch\",\"offset-from-origin\":{\"seconds\":1790000000,\"cycles\":0}}\n" + + "\u001e{\"type\":\"data-stream-class\",\"id\":1,\"default-clock-class-id\":\"monotonic\",\"packet-context-field-class\":\"pkt-ctx\",\"event-record-header-field-class\":\"er-header-compact\",\"event-record-common-context-field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"pid\",\"field-class\":\"s32\"},{\"name\":\"tid\",\"field-class\":\"s32\"},{\"name\":\"procname\",\"field-class\":\"comm\"},{\"name\":\"prio\",\"field-class\":\"s32\"}]}}\n" + + "\u001e{\"type\":\"data-stream-class\",\"id\":0,\"default-clock-class-id\":\"monotonic\",\"packet-context-field-class\":\"pkt-ctx\",\"event-record-header-field-class\":\"er-header-compact\",\"event-record-common-context-field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"perf_cpu_cycles\",\"field-class\":\"u64\"},{\"name\":\"perf_cpu_instructions\",\"field-class\":\"u64\"},{\"name\":\"pid\",\"field-class\":\"s32\"},{\"name\":\"tid\",\"field-class\":\"s32\"},{\"name\":\"procname\",\"field-class\":\"comm\"},{\"name\":\"prio\",\"field-class\":\"s32\"}]}}\n" + + "\u001e{\"type\":\"event-record-class\",\"data-stream-class-id\":1,\"id\":0,\"name\":\"sched_wakeup\",\"payload-field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"comm\",\"field-class\":\"comm\"},{\"name\":\"tid\",\"field-class\":\"s32\"},{\"name\":\"prio\",\"field-class\":\"s32\"},{\"name\":\"target_cpu\",\"field-class\":\"s32\"}]}}\n" + + "\u001e{\"type\":\"event-record-class\",\"data-stream-class-id\":0,\"id\":0,\"name\":\"sched_switch\",\"payload-field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"prev_comm\",\"field-class\":\"comm\"},{\"name\":\"prev_tid\",\"field-class\":\"s32\"},{\"name\":\"prev_prio\",\"field-class\":\"s32\"},{\"name\":\"prev_state\",\"field-class\":{\"type\":\"fixed-length-signed-integer\",\"length\":64,\"alignment\":8,\"byte-order\":\"little-endian\"}},{\"name\":\"next_comm\",\"field-class\":\"comm\"},{\"name\":\"next_tid\",\"field-class\":\"s32\"},{\"name\":\"next_prio\",\"field-class\":\"s32\"}]}}\n"; + + private static string tracePath; + private static RuntimeExecutionResults results; + private static DataCookerPath threadCookerPath; + private static DataCookerPath genericCookerPath; + + [ClassInitialize] + public static void ProcessTrace(TestContext context) + { + tracePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".ctf"); + WriteTrace(tracePath); + + var runtime = Engine.Create(new FileDataSource(tracePath)); + + var threadCooker = new LTTngThreadDataCooker(); + threadCookerPath = threadCooker.Path; + runtime.EnableCooker(threadCookerPath); + + var genericCooker = new LTTngGenericEventDataCooker(); + genericCookerPath = genericCooker.Path; + runtime.EnableCooker(genericCookerPath); + + runtime.EnableTable(ExecutionEventTable.TableDescriptor); + + results = runtime.Process(); + } + + [ClassCleanup] + public static void Cleanup() + { + if (tracePath != null && File.Exists(tracePath)) + { + File.Delete(tracePath); + } + } + + [TestMethod] + public void ProcessesAllChannels() + { + Assert.AreEqual(0, results.ProcessingErrors.Count(), string.Join(Environment.NewLine, results.ProcessingErrors.Select(e => e.ProcessFault?.ToString()))); + + var events = results.QueryOutput>( + new DataOutputPath(genericCookerPath, nameof(LTTngGenericEventDataCooker.Events))); + + var names = Enumerable.Range(0, (int)events.Count).Select(i => events[i].EventName).ToList(); + Assert.AreEqual(3, names.Count(n => n == "sched_switch")); + Assert.AreEqual(1, names.Count(n => n == "sched_wakeup")); + + var wakeup = Enumerable.Range(0, (int)events.Count).Select(i => events[i]).Single(e => e.EventName == "sched_wakeup"); + Assert.AreEqual("taskA", wakeup.FieldValues[wakeup.FieldNames.IndexOf("_comm")]); + } + + [TestMethod] + public void ExecutionEventsHavePerformanceCounterDeltas() + { + var executionEvents = results.QueryOutput>( + new DataOutputPath(threadCookerPath, nameof(LTTngThreadDataCooker.ExecutionEvents))); + + Assert.AreEqual(2, executionEvents.Count); + + var taskA = executionEvents.Single(e => e.NextTid == 100); + Assert.AreEqual("taskA", taskA.NextImage); + Assert.AreEqual(500, taskA.PerformanceCountersDiffByName["_perf_cpu_cycles"]); + Assert.AreEqual(600, taskA.PerformanceCountersDiffByName["_perf_cpu_instructions"]); + + var idle = executionEvents.Single(e => e.NextTid == 0); + Assert.AreEqual(300, idle.PerformanceCountersDiffByName["_perf_cpu_cycles"]); + Assert.AreEqual(400, idle.PerformanceCountersDiffByName["_perf_cpu_instructions"]); + + var table = results.BuildTable(ExecutionEventTable.TableDescriptor); + var cycles = table.Columns.Single(c => c.Configuration.Metadata.Name == "CPU Cycle"); + var instructions = table.Columns.Single(c => c.Configuration.Metadata.Name == "Instruction Count"); + CollectionAssert.AreEquivalent(new long[] { 500, 300 }, Enumerable.Range(0, table.RowCount).Select(i => (long)cycles.Project(i)).ToArray()); + CollectionAssert.AreEquivalent(new long[] { 600, 400 }, Enumerable.Range(0, table.RowCount).Select(i => (long)instructions.Project(i)).ToArray()); + } + + private static void WriteTrace(string path) + { + using (var archive = ZipFile.Open(path, ZipArchiveMode.Create)) + { + WriteEntry(archive, "trace/kernel/metadata", Encoding.UTF8.GetBytes(Metadata)); + + // Channel "sw-chan" (stream class 0) on CPU 0: idle -> taskA -> idle -> taskA. + WriteEntry(archive, "trace/kernel/sw-chan_0", Packet(0, 1000, 3000, + SchedSwitch(1000, cycles: 1000, instructions: 2000, prevTid: 0, prevComm: "swapper/0", nextTid: 100, nextComm: "taskA"), + SchedSwitch(2000, cycles: 1500, instructions: 2600, prevTid: 100, prevComm: "taskA", nextTid: 0, nextComm: "swapper/0"), + SchedSwitch(3000, cycles: 1800, instructions: 3000, prevTid: 0, prevComm: "swapper/0", nextTid: 100, nextComm: "taskA"))); + + // Channel "other-chan" (stream class 1) on CPU 0: its event id 0 is sched_wakeup, not sched_switch. + WriteEntry(archive, "trace/kernel/other-chan_0", Packet(1, 1500, 1500, SchedWakeup(1500, tid: 100, comm: "taskA"))); + } + } + + private static void WriteEntry(ZipArchive archive, string name, byte[] content) + { + using (var stream = archive.CreateEntry(name).Open()) + { + stream.Write(content, 0, content.Length); + } + } + + private static byte[] Packet(uint streamId, ulong begin, ulong end, params byte[][] events) + { + const int headerAndContextSize = 4 + 16 + 4 + (5 * 8) + 4; + ulong sizeInBits = (ulong)(headerAndContextSize + events.Sum(e => e.Length)) * 8; + + var stream = new MemoryStream(); + var writer = new BinaryWriter(stream); + writer.Write(0xC1FC1FC1u); + writer.Write(new byte[16]); + writer.Write(streamId); + writer.Write(begin); + writer.Write(end); + writer.Write(sizeInBits); + writer.Write(sizeInBits); + writer.Write(0ul); + writer.Write(0u); + foreach (var e in events) + { + writer.Write(e); + } + + return stream.ToArray(); + } + + private static byte[] SchedSwitch(uint timestamp, ulong cycles, ulong instructions, int prevTid, string prevComm, int nextTid, string nextComm) + { + var stream = new MemoryStream(); + var writer = new BinaryWriter(stream); + WriteCompactHeader(writer, 0, timestamp); + writer.Write(cycles); + writer.Write(instructions); + WriteProcessContext(writer, prevTid, prevComm); + writer.Write(Comm(prevComm)); + writer.Write(prevTid); + writer.Write(20); + writer.Write(0L); + writer.Write(Comm(nextComm)); + writer.Write(nextTid); + writer.Write(20); + return stream.ToArray(); + } + + private static byte[] SchedWakeup(uint timestamp, int tid, string comm) + { + var stream = new MemoryStream(); + var writer = new BinaryWriter(stream); + WriteCompactHeader(writer, 0, timestamp); + WriteProcessContext(writer, 0, "swapper/0"); + writer.Write(Comm(comm)); + writer.Write(tid); + writer.Write(20); + writer.Write(0); + return stream.ToArray(); + } + + private static void WriteCompactHeader(BinaryWriter writer, uint id, uint timestamp) + { + // 5-bit event id followed by a 27-bit timestamp, packed least significant bit first. + writer.Write(id | (timestamp << 5)); + } + + private static void WriteProcessContext(BinaryWriter writer, int tid, string procname) + { + writer.Write(tid); + writer.Write(tid); + writer.Write(Comm(procname)); + writer.Write(20); + } + + private static byte[] Comm(string value) + { + var bytes = new byte[16]; + Encoding.UTF8.GetBytes(value, 0, value.Length, bytes, 0); + return bytes; + } + } +} diff --git a/LTTngDataExtUnitTest/LTTngEventKindTests.cs b/LTTngDataExtUnitTest/LTTngEventKindTests.cs new file mode 100644 index 0000000..672d502 --- /dev/null +++ b/LTTngDataExtUnitTest/LTTngEventKindTests.cs @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace LTTngDataExtUnitTest +{ + /// + /// Generic events must be named from their own trace's metadata, even when traces share a domain and their + /// stream and event ids overlap (e.g. per-UID and per-PID UST buffers), and when traces are processed one after + /// another in the same process. + /// + [TestClass] + public class LTTngEventKindTests + { + private const string ValuePayload = @"[{""name"":""value"",""field-class"":""s32""}]"; + + [TestMethod] + public void TracesInOneInputWithOverlappingEventIds() + { + var events = Process( + ("trace/ust/uid/1000/64-bit", "app_a_event", 1000, 1), + ("trace/ust/pid/app-42/64-bit", "app_b_event", 2000, 2)); + + CollectionAssert.AreEquivalent(new[] { "app_a_event = 1", "app_a_event = 1", "app_b_event = 2", "app_b_event = 2" }, events, string.Join(", ", events)); + } + + [TestMethod] + public void TracesProcessedOneAfterAnother() + { + var first = Process(("trace/ust/uid/1000/64-bit", "first_event", 1000, 1)); + var second = Process(("trace/ust/uid/1000/64-bit", "second_event", 1000, 2)); + + CollectionAssert.AreEqual(new[] { "first_event = 1", "first_event = 1" }, first, string.Join(", ", first)); + CollectionAssert.AreEqual(new[] { "second_event = 2", "second_event = 2" }, second, string.Join(", ", second)); + } + + // Each trace has one event record class (id 0) and two events of it. + private static List Process(params (string Directory, string EventName, uint Timestamp, int Value)[] traces) + { + var events = Ctf2TestTraces.ProcessGenericEvents(traces.Select(trace => ( + trace.Directory, + Ctf2TestTraces.Metadata("ust", Ctf2TestTraces.EventRecordClass(0, trace.EventName, ValuePayload)), + Ctf2TestTraces.Packet( + trace.Timestamp, + trace.Timestamp + 100, + Ctf2TestTraces.Event(0, trace.Timestamp, BitConverter.GetBytes(trace.Value)), + Ctf2TestTraces.Event(0, trace.Timestamp + 100, BitConverter.GetBytes(trace.Value))))).ToArray()); + + return events.Select(e => $"{e.EventName} = {e.FieldValues[e.FieldNames.IndexOf("_value")]}").ToList(); + } + } +} \ No newline at end of file diff --git a/LTTngDataExtensions/DataOutputTypes/LTTngGenericEvent.cs b/LTTngDataExtensions/DataOutputTypes/LTTngGenericEvent.cs index 3b46c1d..0a384c7 100644 --- a/LTTngDataExtensions/DataOutputTypes/LTTngGenericEvent.cs +++ b/LTTngDataExtensions/DataOutputTypes/LTTngGenericEvent.cs @@ -3,10 +3,12 @@ using System; using System.Collections.Generic; +using System.Runtime.CompilerServices; using System.Text.RegularExpressions; using CtfPlayback; using CtfPlayback.FieldValues; using LTTngCds.CookerData; +using LTTngCds.CtfExtensions.DescriptorInterfaces; using Microsoft.Performance.SDK; namespace LTTngDataExtensions.DataOutputTypes @@ -74,17 +76,41 @@ public EventKind(LTTngContext context, uint id, string name, IReadOnlyList RegisteredKinds = new Dictionary(); private static readonly Regex TraceLoggingEventRegex = new Regex("^(?[a-zA-Z_.0-9]+):(?[a-zA-Z_.0-9]+);(?.+);$"); + // Event ids are only unique within one stream of one trace, and a single input can contain several traces of + // the same domain (e.g. per-UID and per-PID UST buffers). The event descriptor identifies the event class + // itself. Entries are released with the trace's metadata, so traces processed later or concurrently never + // share kinds. + private static readonly ConditionalWeakTable KindsByEventClass = new ConditionalWeakTable(); + + internal static EventKind GetKind(LTTngEvent data, LTTngContext context, IReadOnlyList fields) + { + if (KindsByEventClass.TryGetValue(data.EventDescriptor, out var kind)) + { + return kind; + } + + return AddKind(data, context, fields); + } + + [Obsolete("Event ids are not unique across streams or traces. LTTngGenericEvent identifies event kinds by event class instead.")] public static bool TryGetRegisteredKind(string domain, uint id, out EventKind kind) { return RegisteredKinds.TryGetValue(new Key(domain, id), out kind); } + [Obsolete("Event ids are not unique across streams or traces. LTTngGenericEvent identifies event kinds by event class instead.")] public static EventKind RegisterKind(LTTngContext context, uint id, string name, IReadOnlyList fields) { EventKind kind = new EventKind(context, id, name, fields); RegisteredKinds.Add(new Key(context.Domain, id), kind); return kind; } + + private static EventKind AddKind(LTTngEvent data, LTTngContext context, IReadOnlyList fields) + { + // If another thread adds the same event class first, its kind is returned. + return KindsByEventClass.GetValue(data.EventDescriptor, _ => new EventKind(context, data.Id, data.Name, fields)); + } } public struct LTTngGenericEvent @@ -103,10 +129,7 @@ public LTTngGenericEvent(LTTngEvent data, LTTngContext context) this.CpuId = context.CurrentCpu; this.DiscardedEvents = data.DiscardedEvents; - if (!EventKind.TryGetRegisteredKind(context.Domain, data.Id, out this.kind)) - { - this.kind = EventKind.RegisterKind(context, data.Id, data.Name, payload.Fields); - } + this.kind = EventKind.GetKind(data, context, payload.Fields); // As this is being written, all columns are of type 'T', so all rows are the same. For generic events, // where columns have different types for different rows, this means everything becomes a string. @@ -124,9 +147,15 @@ public LTTngGenericEvent(LTTngEvent data, LTTngContext context) fieldCount += data.StreamDefinedEventContext.Fields.Count; } + if (data.SpecificContext != null) + { + fieldCount += data.SpecificContext.Fields.Count; + } + this.FieldNames = new List(fieldCount); this.FieldValues = new List(fieldCount); + // Fields in the order they are recorded: stream context, event specific context, then payload. if (data.StreamDefinedEventContext != null) { foreach (var field in data.StreamDefinedEventContext.Fields) @@ -135,6 +164,16 @@ public LTTngGenericEvent(LTTngEvent data, LTTngContext context) this.FieldValues.Add(field.GetValueAsString()); } } + + if (data.SpecificContext != null) + { + foreach (var field in data.SpecificContext.Fields) + { + this.FieldNames.Add(field.FieldName.ToString()); + this.FieldValues.Add(field.GetValueAsString()); + } + } + foreach (var field in payload.Fields) { this.FieldNames.Add(field.FieldName.ToString()); diff --git a/LTTngDataExtensions/SourceDataCookers/LTTngGenericEventDataCooker.cs b/LTTngDataExtensions/SourceDataCookers/LTTngGenericEventDataCooker.cs index e4ae374..40b76c0 100644 --- a/LTTngDataExtensions/SourceDataCookers/LTTngGenericEventDataCooker.cs +++ b/LTTngDataExtensions/SourceDataCookers/LTTngGenericEventDataCooker.cs @@ -45,10 +45,12 @@ public override DataProcessingResult CookDataElement( { try { - Events.AddEvent(new LTTngGenericEvent(data, context)); + var genericEvent = new LTTngGenericEvent(data, context); + Events.AddEvent(genericEvent); + // Generic events list their context fields as well as their payload fields. this.MaximumEventFieldCount = - Math.Max(data.Payload.Fields.Count, this.MaximumEventFieldCount); + Math.Max(genericEvent.FieldNames.Count, this.MaximumEventFieldCount); } catch (CtfPlaybackException e) { diff --git a/LinuxTraceLogCapture.md b/LinuxTraceLogCapture.md index 5371774..fdf5df0 100644 --- a/LinuxTraceLogCapture.md +++ b/LinuxTraceLogCapture.md @@ -44,7 +44,9 @@ Supports: - Block IO / Disk Activity - Diagnostic Messages -Once you have everything set up you just need to decide what kind of information you are looking for and begin tracing. +Traces in both the CTF 1.8 and CTF 2 formats are supported (CTF 2 is the default trace format since LTTng 2.15). + +Once you have everything set up you just need to decide what kind of information you are looking for and begin tracing. In this example we are looking at process scheduler events. We might use this to determine process lifetime and identify dependencies. You can learn more about what kind of "events" you can enable [here](https://lttng.org/man/1/lttng-enable-event/v2.8/). ```bash @@ -85,6 +87,12 @@ $ sudo lttng add-context --kernel --channel=channel0 --type=pid $ sudo lttng add-context --kernel --channel=channel0 --type=procname ``` +### Optionally, add CPU performance counters: +The Execution Events table shows how much each counter changed while a thread was running (Instruction Count, CPU Cycle, LLC Misses, Cache Misses). +```bash +$ sudo lttng add-context --kernel --channel=channel0 --type=perf:cpu:instructions --type=perf:cpu:cycles --type=perf:cpu:cache-misses --type=perf:cpu:LLC-load-misses +``` + ### Start the recording: ```bash $ sudo lttng start