diff --git a/docs/Features/Packages/Import-export tool.md b/docs/Features/Packages/Import-export tool.md index 5cfdcce0..44489d51 100644 --- a/docs/Features/Packages/Import-export tool.md +++ b/docs/Features/Packages/Import-export tool.md @@ -55,7 +55,7 @@ python impexp.py import MyProject.twinproj unpacked/ Scans a directory tree and writes a `.twinproj` or `.twinpack` binary. The directory name becomes the root entry name in the output file. Well-known directory and file names (`Sources`, `Resources`, `Settings`, etc.) are -tagged with the correct `mark2` category values automatically. +tagged with the correct `category` values automatically. ``` node impexp.mjs export unpacked/ MyProject.twinproj @@ -81,9 +81,9 @@ Importing and re-exporting a binary file preserves all file contents byte-for-byte. The following metadata fields are reset to defaults on a disk round-trip (they are not stored on the filesystem): -- **mark1** (revision counter) -- directories get `0x0000`; files get - `0x0002`. -- **Revision entries** -- always written as zero. +- **revision counter** -- directories get `0x0000`; files get `0x0002`. +- **flags** -- always written as zero (no flags set). +- **Revision trailer entries** -- always written as zero. - **Entry order** -- directories first, then files, sorted alphabetically within each group. diff --git a/docs/Features/Packages/TWINPACK file format.md b/docs/Features/Packages/TWINPACK file format.md index 66bb40df..76c9427f 100644 --- a/docs/Features/Packages/TWINPACK file format.md +++ b/docs/Features/Packages/TWINPACK file format.md @@ -38,33 +38,26 @@ A length-prefixed byte string. Encoding is UTF-8 for filenames and text content ## Entry structure -Every node in the tree — the root, directories, and files — shares a common header: +Every node in the tree — the root, directories, and files — shares the same common header. The first 2 bytes carry one of two meanings depending on position: at the root entry they hold the file format version; everywhere else they hold the entry kind. -| Offset | Size | Type | Field | Description | -|--------|------|-----------|---------|-------------| -| +0 | 2 | int16 | `kind` | Entry type (see below). | -| +2 | var | LenString | `name` | Entry name (filename or folder name). | -| +2+var | 2 | uint16 | `mark1` | Revision counter (see [mark1](#mark1-revision-counter)). | -| ... | 10 | byte[] | `pad` | Reserved. Always observed as all zeros. | -| ... | 1 | uint8 | `mark2` | Category tag (see [mark2](#mark2-category-tag)). | +| Offset | Size | Type | Field | Description | +|--------|------|-----------|------------|-------------| +| +0 | 2 | int16 | `kind` | At the root: file format version (currently `1`). Everywhere else: entry kind (`1` = file, `2` = directory). | +| +2 | var | LenString | `name` | Entry name (filename or folder name). | +| +2+var | 8 | uint64 | `revision` | Revision counter (see [revision](#revision-counter)). | +| ... | 4 | uint32 | `flags` | File-system flags bitmask (see [flags](#flags)). | +| ... | 1 | uint8 | `category` | Category tag (see [category](#category-tag)). | After this common header, the entry body depends on whether the entry is a **file** or a **directory**. ### Determining entry type -The root entry is always the first entry parsed. It is always a directory (even though it has `kind=1`), because the format uses positional logic: +The root entry is always the first entry parsed. It is always a directory — its 2-byte field is the file format version, not a kind tag, and its value (currently `1`) coincides with the file-kind value but should not be read as one. Every entry after the root is determined by its `kind`: -- **Directory** — the root entry, or any entry with `kind != 1`. - Body: child count followed by child entries. -- **File** — any non-root entry with `kind == 1`. +- **File** — `kind == 1`. Body: content blob followed by a revision trailer. - -Observed `kind` values: - -| kind | Meaning | -|------|---------| -| 1 | File (or root directory — always the first entry) | -| 2 | Directory | +- **Directory** — `kind == 2`. + Body: child count followed by child entries. ### Directory body @@ -89,9 +82,9 @@ The `revisionCount` field is 0 for the vast majority of files, making the file b ## Field details -### mark1 (revision counter) +### revision counter -For files, `mark1` is a revision counter that starts at a low value and increments with each edit inside the IDE. For the root entry and directories it is always 0. +For files, `revision` is a 64-bit counter that starts at a low value and increments with each edit inside the IDE. For the root entry and directories it is always 0. | Context | Typical values | |---------|----------------| @@ -99,19 +92,38 @@ For files, `mark1` is a revision counter that starts at a low value and incremen | Heavily edited file | `0x17D5`, `0x1AA0` | | Root and directories | `0x0000` | -### mark2 (category tag) +Only the low 16 bits have been observed to vary in real-world files; the upper 48 bits are always zero in practice. + +### flags + +A 32-bit bitmask describing file-system-level properties of the entry. Every entry observed so far has `flags == 0`, but the IDE recognises the following bits: + +| Bit value | Name | Meaning | +|--------------|---------------|---------| +| `0x00000000` | `None` | Default — no flags set. | +| `0x00000001` | `Hidden` | Hidden from the user, but accessible via the VFS. | +| `0x00000002` | `SuperHidden` | Not accessible via the VFS — internal only. | +| `0x00000004` | `Virtual` | Virtual items are skipped during serialization. | + +Other bits are reserved. + +### category tag Encodes the semantic role of the entry within the project: -| mark2 | Entry name | Meaning | -|-------|-------------------------|---------| -| 0x00 | *(various)* | Default. Used for the root, most files, and resource subdirectories (`BITMAP`, `ICON`, `MANIFEST`). | -| 0x02 | `Resources` | Resource directory. | -| 0x03 | `Sources` | Source code directory. | -| 0x04 | `Settings` | Project settings file (JSON). | -| 0x05 | `ImportedTypeLibraries` | Imported type library directory. | -| 0x06 | `Miscellaneous` | Miscellaneous files directory (screenshots, etc.). | -| 0x07 | `Packages` | Package references directory. | +| category | Entry name | Meaning | +|----------|-------------------------|---------| +| 0x00 | *(various)* | Default. Used for the root, most files, and resource subdirectories (`BITMAP`, `ICON`, `MANIFEST`). | +| 0x01 | `References` | References directory. Always virtual — see note below. | +| 0x02 | `Resources` | Resource directory. | +| 0x03 | `Sources` | Source code directory. | +| 0x04 | `Settings` | Project settings file (JSON). | +| 0x05 | `ImportedTypeLibraries` | Imported type library directory. | +| 0x06 | `Miscellaneous` | Miscellaneous files directory (screenshots, etc.). | +| 0x07 | `Packages` | Package references directory. | + +> [!NOTE] +> The `References` directory (category `0x01`) is a virtual folder — it carries the [`Virtual`](#flags) flag and is skipped during serialization, so it never appears in saved `.twinproj` or `.twinpack` files. The IDE materialises it at runtime from the project's references list. ## Differences between .twinproj and .twinpack @@ -120,7 +132,6 @@ Both formats use the identical binary structure. The differences are in which e | Entry | .twinproj | .twinpack | |------------------------|-----------|-----------| | `.meta` file | Yes | No | -| `References` directory | Sometimes | No | | `CHANGELOG.md` | Sometimes | Sometimes | | `LICENCE.md` | Sometimes | Sometimes | | `Settings` file | Yes | Yes | @@ -128,51 +139,53 @@ Both formats use the identical binary structure. The differences are in which e | `Resources` directory | Yes | Yes | | `Packages` directory | Yes | Yes | +The `References` directory (category `0x01`) is virtual and is omitted from both formats during serialization — see [category tag](#category-tag). + ### .meta file Present only in `.twinproj` files. Contains JSON storing the user's IDE layout preferences — expanded folders, open editors, watch list, and outline-panel options. This file is stripped when the IDE generates a `.twinpack` for distribution. ### Settings file -Always present (`mark2 = 0x04`). Contains JSON with project configuration including the build type, references, version numbers, and other project settings. +Always present (`category = 0x04`). Contains JSON with project configuration including the build type, references, version numbers, and other project settings. ## Typical tree structures ### .twinproj (Standard EXE project) ``` -ROOT "NewProject" (kind=1, mark2=0x00) - DIR "Miscellaneous" (kind=2, mark2=0x06) - DIR "Packages" (kind=2, mark2=0x07) - DIR "ImportedTypeLibraries" (kind=2, mark2=0x05) - DIR "Resources" (kind=2, mark2=0x02) - DIR "ICON" (kind=2, mark2=0x00) - FILE "twinBASIC.ico" (kind=1, mark2=0x00) - DIR "Sources" (kind=2, mark2=0x03) - FILE "Form1.tbform" (kind=1, mark2=0x00) - FILE "Form1.twin" (kind=1, mark2=0x00) - FILE "Settings" (kind=1, mark2=0x04) - FILE ".meta" (kind=1, mark2=0x00) +ROOT "NewProject" (version=1, category=0x00) + DIR "Miscellaneous" (kind=2, category=0x06) + DIR "Packages" (kind=2, category=0x07) + DIR "ImportedTypeLibraries" (kind=2, category=0x05) + DIR "Resources" (kind=2, category=0x02) + DIR "ICON" (kind=2, category=0x00) + FILE "twinBASIC.ico" (kind=1, category=0x00) + DIR "Sources" (kind=2, category=0x03) + FILE "Form1.tbform" (kind=1, category=0x00) + FILE "Form1.twin" (kind=1, category=0x00) + FILE "Settings" (kind=1, category=0x04) + FILE ".meta" (kind=1, category=0x00) ``` ### .twinpack (distributed package) ``` -ROOT "CustomControlsPackage" (kind=1, mark2=0x00) - FILE "CHANGELOG.md" (kind=1, mark2=0x00) - FILE "LICENCE.md" (kind=1, mark2=0x00) - DIR "Miscellaneous" (kind=2, mark2=0x06) - FILE "frmTextbox.png" (kind=1, mark2=0x00) - DIR "ImportedTypeLibraries" (kind=2, mark2=0x05) - FILE "Settings" (kind=1, mark2=0x04) - DIR "Sources" (kind=2, mark2=0x03) - FILE "WaynesGrid.twin" (kind=1, mark2=0x00) - DIR "Resources" (kind=2, mark2=0x02) - DIR "MANIFEST" (kind=2, mark2=0x00) - FILE "#1.xml" (kind=1, mark2=0x00) - DIR "BITMAP" (kind=2, mark2=0x00) - FILE "twinBASIC.bmp" (kind=1, mark2=0x00) - DIR "Packages" (kind=2, mark2=0x07) +ROOT "CustomControlsPackage" (version=1, category=0x00) + FILE "CHANGELOG.md" (kind=1, category=0x00) + FILE "LICENCE.md" (kind=1, category=0x00) + DIR "Miscellaneous" (kind=2, category=0x06) + FILE "frmTextbox.png" (kind=1, category=0x00) + DIR "ImportedTypeLibraries" (kind=2, category=0x05) + FILE "Settings" (kind=1, category=0x04) + DIR "Sources" (kind=2, category=0x03) + FILE "WaynesGrid.twin" (kind=1, category=0x00) + DIR "Resources" (kind=2, category=0x02) + DIR "MANIFEST" (kind=2, category=0x00) + FILE "#1.xml" (kind=1, category=0x00) + DIR "BITMAP" (kind=2, category=0x00) + FILE "twinBASIC.bmp" (kind=1, category=0x00) + DIR "Packages" (kind=2, category=0x07) ``` ## Notes diff --git a/indexer/sample.twinpack b/indexer/sample.twinpack new file mode 100644 index 00000000..841b6e28 Binary files /dev/null and b/indexer/sample.twinpack differ diff --git a/scripts/impexp.mjs b/scripts/impexp.mjs index 9ecc16a0..5bf802c5 100644 --- a/scripts/impexp.mjs +++ b/scripts/impexp.mjs @@ -17,10 +17,37 @@ import os from 'node:os'; import { fileURLToPath } from 'node:url'; const MAGIC = 0xEA0BA51C; +const FORMAT_VERSION = 1; -const DIR_MARK2 = { - Resources: 0x02, Sources: 0x03, ImportedTypeLibraries: 0x05, - Miscellaneous: 0x06, Packages: 0x07, +const FLAGS = { + None: 0x00000000, + Hidden: 0x00000001, + SuperHidden: 0x00000002, + Virtual: 0x00000004, +}; + +const CATEGORY = { + Default: 0x00, + References: 0x01, // always virtual; never present in serialized files + Resources: 0x02, + Sources: 0x03, + Settings: 0x04, + ImportedTypeLibraries: 0x05, + Miscellaneous: 0x06, + Packages: 0x07, +}; + +// Well-known entry names that get a non-default category on export. +// References is intentionally excluded -- it is materialised virtually by +// the IDE, never serialized, and tagging an on-disk folder with category +// 0x01 would confuse the IDE on import. +const CATEGORY_BY_NAME = { + Resources: CATEGORY.Resources, + Sources: CATEGORY.Sources, + Settings: CATEGORY.Settings, + ImportedTypeLibraries: CATEGORY.ImportedTypeLibraries, + Miscellaneous: CATEGORY.Miscellaneous, + Packages: CATEGORY.Packages, }; // -------------------------- Parser (binary -> tree) -------------------------- @@ -32,8 +59,8 @@ function parse(buffer) { let pos = 0; let entryCount = 0; + function readU64() { const v = view.getBigUint64(pos, true); pos += 8; return Number(v); } function readU32() { const v = view.getUint32(pos, true); pos += 4; return v; } - function readU16() { const v = view.getUint16(pos, true); pos += 2; return v; } function readI16() { const v = view.getInt16(pos, true); pos += 2; return v; } function readU8() { const v = view.getUint8(pos); pos += 1; return v; } function readStr() { @@ -57,25 +84,32 @@ function parse(buffer) { `expected 0x${MAGIC.toString(16).padStart(8, '0').toUpperCase()}`); function readEntry() { + // At the root this 2-byte field is the file format version; everywhere + // else it is the entry kind (1 = file, 2 = directory). const kind = readI16(); - const name = readStr(); - const mark1 = readU16(); - pos += 10; // reserved padding -- always zeros - const mark2 = readU8(); entryCount++; + const isRoot = (entryCount === 1); + if (isRoot && kind !== FORMAT_VERSION) + throw new Error( + `Unsupported file format version: ${kind}, expected ${FORMAT_VERSION}`); + + const name = readStr(); + const revision = readU64(); + const flags = readU32(); + const category = readU8(); - if (kind === 1 && entryCount > 1) { + if (kind === 1 && !isRoot) { const content = readBlob(); const revisionCount = readU32(); const revisions = []; for (let i = 0; i < revisionCount; i++) revisions.push(readU32()); - return { kind: 'file', name, mark1, mark2, content, revisions }; + return { kind: 'file', name, revision, flags, category, content, revisions }; } const count = readU32(); const children = []; for (let i = 0; i < count; i++) children.push(readEntry()); - return { kind: 'directory', name, mark1, mark2, children }; + return { kind: 'directory', name, revision, flags, category, children }; } return readEntry(); @@ -87,9 +121,9 @@ function serialize(root) { const chunks = []; const encoder = new TextEncoder(); + function writeU64(v) { const b = Buffer.alloc(8); b.writeBigUInt64LE(BigInt(v)); chunks.push(b); } function writeU32(v) { const b = Buffer.alloc(4); b.writeUInt32LE(v); chunks.push(b); } function writeI16(v) { const b = Buffer.alloc(2); b.writeInt16LE(v); chunks.push(b); } - function writeU16(v) { const b = Buffer.alloc(2); b.writeUInt16LE(v); chunks.push(b); } function writeU8(v) { chunks.push(Buffer.from([v])); } function writeStr(s) { const e = encoder.encode(s); @@ -111,19 +145,20 @@ function serialize(root) { if (entry.kind === 'file' && !isRoot) { writeI16(1); writeStr(entry.name); - writeU16(entry.mark1 ?? 0x0002); - chunks.push(Buffer.alloc(10)); - writeU8(entry.mark2 ?? 0x00); + writeU64(entry.revision ?? 0x0002); + writeU32(entry.flags ?? FLAGS.None); + writeU8(entry.category ?? 0x00); writeBlob(entry.content); const revs = entry.revisions ?? []; writeU32(revs.length); for (const r of revs) writeU32(r); } else { - writeI16(isRoot ? 1 : 2); + // Root entry writes the format version; non-root directory writes kind=2. + writeI16(isRoot ? FORMAT_VERSION : 2); writeStr(entry.name); - writeU16(entry.mark1 ?? 0x0000); - chunks.push(Buffer.alloc(10)); - writeU8(entry.mark2 ?? 0x00); + writeU64(entry.revision ?? 0x0000); + writeU32(entry.flags ?? FLAGS.None); + writeU8(entry.category ?? 0x00); const children = entry.children ?? []; writeU32(children.length); for (const child of children) writeEntry(child); @@ -162,9 +197,8 @@ function doImport(inputPath, outputDir, { quiet = false } = {}) { // -------------------------- Export (disk -> binary) -------------------------- -function mark2For(name, isDir) { - if (isDir) return DIR_MARK2[name] ?? 0x00; - return name === 'Settings' ? 0x04 : 0x00; +function categoryFor(name) { + return CATEGORY_BY_NAME[name] ?? CATEGORY.Default; } function buildTree(dirPath) { @@ -180,14 +214,14 @@ function buildTree(dirPath) { for (const f of files) { children.push({ kind: 'file', name: f.name, - mark1: 0x0002, mark2: mark2For(f.name, false), + revision: 0x0002, flags: FLAGS.None, category: categoryFor(f.name), content: fs.readFileSync(path.join(dirPath, f.name)), revisions: [], }); } return { kind: 'directory', name, - mark1: 0x0000, mark2: mark2For(name, true), + revision: 0x0000, flags: FLAGS.None, category: categoryFor(name), children, }; } @@ -293,7 +327,7 @@ function selfTest() { }); test('Empty project round-trip', () => { - const tree = { kind: 'directory', name: 'Empty', mark1: 0, mark2: 0, children: [] }; + const tree = { kind: 'directory', name: 'Empty', revision: 0, flags: 0, category: 0, children: [] }; const rt = parse(serialize(tree)); eq(rt.name, 'Empty', 'name'); eq(rt.children.length, 0, 'children'); @@ -302,8 +336,8 @@ function selfTest() { test('Single-file project round-trip', () => { const content = Buffer.from('Hello twinBASIC'); const tree = { - kind: 'directory', name: 'Mini', mark1: 0, mark2: 0, - children: [{ kind: 'file', name: 'test.twin', mark1: 2, mark2: 0, content, revisions: [] }], + kind: 'directory', name: 'Mini', revision: 0, flags: 0, category: 0, + children: [{ kind: 'file', name: 'test.twin', revision: 2, flags: 0, category: 0, content, revisions: [] }], }; const rt = parse(serialize(tree)); eq(rt.children.length, 1, 'children'); @@ -312,6 +346,20 @@ function selfTest() { throw new Error('content mismatch'); }); + test('Flags field preserved on round-trip', () => { + const tree = { + kind: 'directory', name: 'WithFlags', revision: 0, flags: FLAGS.Hidden, category: 0, + children: [{ + kind: 'file', name: 'h.twin', + revision: 2, flags: FLAGS.Hidden | FLAGS.Virtual, category: 0, + content: Buffer.from('x'), revisions: [], + }], + }; + const rt = parse(serialize(tree)); + eq(rt.flags, FLAGS.Hidden, 'root flags'); + eq(rt.children[0].flags, FLAGS.Hidden | FLAGS.Virtual, 'file flags'); + }); + test('Bad magic rejected', () => { try { parse(Buffer.from('not a twinpack!!')); throw new Error('should have thrown'); } catch (e) { if (!e.message.includes('Bad magic')) throw e; } diff --git a/scripts/impexp.py b/scripts/impexp.py index 13d9ffaa..4f7cc97c 100644 --- a/scripts/impexp.py +++ b/scripts/impexp.py @@ -16,10 +16,33 @@ import sys MAGIC = 0xEA0BA51C - -DIR_MARK2 = { - 'Resources': 0x02, 'Sources': 0x03, 'ImportedTypeLibraries': 0x05, - 'Miscellaneous': 0x06, 'Packages': 0x07, +FORMAT_VERSION = 1 + +FLAGS_NONE = 0x00000000 +FLAGS_HIDDEN = 0x00000001 +FLAGS_SUPER_HIDDEN = 0x00000002 +FLAGS_VIRTUAL = 0x00000004 + +CATEGORY_DEFAULT = 0x00 +CATEGORY_REFERENCES = 0x01 # always virtual; never present in serialized files +CATEGORY_RESOURCES = 0x02 +CATEGORY_SOURCES = 0x03 +CATEGORY_SETTINGS = 0x04 +CATEGORY_IMPORTED_TYPE_LIBRARIES = 0x05 +CATEGORY_MISCELLANEOUS = 0x06 +CATEGORY_PACKAGES = 0x07 + +# Well-known entry names that get a non-default category on export. +# References is intentionally excluded -- it is materialised virtually by +# the IDE, never serialized, and tagging an on-disk folder with category +# 0x01 would confuse the IDE on import. +CATEGORY_BY_NAME = { + 'Resources': CATEGORY_RESOURCES, + 'Sources': CATEGORY_SOURCES, + 'Settings': CATEGORY_SETTINGS, + 'ImportedTypeLibraries': CATEGORY_IMPORTED_TYPE_LIBRARIES, + 'Miscellaneous': CATEGORY_MISCELLANEOUS, + 'Packages': CATEGORY_PACKAGES, } # -------------------------- Parser (binary -> tree) -------------------------- @@ -28,12 +51,12 @@ def parse(data): pos = [0] + def read_u64(): + v, = struct.unpack_from(' 1: + if kind == 1 and not is_root: content = read_blob() revision_count = read_u32() revisions = [read_u32() for _ in range(revision_count)] - return dict(kind='file', name=name, mark1=mark1, mark2=mark2, + return dict(kind='file', name=name, revision=revision, + flags=flags, category=category, content=content, revisions=revisions) count = read_u32() children = [read_entry() for _ in range(count)] - return dict(kind='directory', name=name, mark1=mark1, mark2=mark2, - children=children) + return dict(kind='directory', name=name, revision=revision, + flags=flags, category=category, children=children) return read_entry() @@ -90,15 +122,15 @@ def read_entry(): def serialize(root): chunks = [] + def write_u64(v): + chunks.append(struct.pack(' binary) -------------------------- -def _mark2_for(name, is_dir): - if is_dir: - return DIR_MARK2.get(name, 0x00) - return 0x04 if name == 'Settings' else 0x00 +def _category_for(name): + return CATEGORY_BY_NAME.get(name, CATEGORY_DEFAULT) def _build_tree(dir_path): @@ -204,12 +236,14 @@ def _build_tree(dir_path): content = fh.read() children.append(dict( kind='file', name=f, - mark1=0x0002, mark2=_mark2_for(f, False), + revision=0x0002, flags=FLAGS_NONE, + category=_category_for(f), content=content, revisions=[], )) return dict( kind='directory', name=name, - mark1=0x0000, mark2=_mark2_for(name, True), + revision=0x0000, flags=FLAGS_NONE, + category=_category_for(name), children=children, ) @@ -351,7 +385,7 @@ def t_disk(): def t_empty(): tree = dict(kind='directory', name='Empty', - mark1=0, mark2=0, children=[]) + revision=0, flags=0, category=0, children=[]) rt = parse(serialize(tree)) eq(rt['name'], 'Empty', 'name') eq(len(rt['children']), 0, 'children') @@ -359,10 +393,12 @@ def t_empty(): def t_single(): content = b'Hello twinBASIC' - tree = dict(kind='directory', name='Mini', mark1=0, mark2=0, + tree = dict(kind='directory', name='Mini', + revision=0, flags=0, category=0, children=[ - dict(kind='file', name='test.twin', mark1=2, - mark2=0, content=content, revisions=[]), + dict(kind='file', name='test.twin', + revision=2, flags=0, category=0, + content=content, revisions=[]), ]) rt = parse(serialize(tree)) eq(len(rt['children']), 1, 'children') @@ -371,6 +407,21 @@ def t_single(): raise AssertionError('content mismatch') test('Single-file project round-trip', t_single) + def t_flags(): + tree = dict( + kind='directory', name='WithFlags', + revision=0, flags=FLAGS_HIDDEN, category=0, + children=[ + dict(kind='file', name='h.twin', + revision=2, flags=FLAGS_HIDDEN | FLAGS_VIRTUAL, + category=0, content=b'x', revisions=[]), + ]) + rt = parse(serialize(tree)) + eq(rt['flags'], FLAGS_HIDDEN, 'root flags') + eq(rt['children'][0]['flags'], + FLAGS_HIDDEN | FLAGS_VIRTUAL, 'file flags') + test('Flags field preserved on round-trip', t_flags) + def t_bad_magic(): try: parse(b'not a twinpack!!')