diff --git a/AGENT.md b/AGENT.md new file mode 120000 index 000000000..708f0d2e3 --- /dev/null +++ b/AGENT.md @@ -0,0 +1 @@ +ARCHITECTURE.md \ No newline at end of file diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index ba924a09a..11f4437b3 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,427 +1,364 @@ -# Qiling Framework Architecture - -Qiling is a cross-platform, multi-architecture binary emulation framework built on top of the [Unicorn](https://www.unicorn-engine.org/) CPU emulation engine. It adds OS-level abstractions (syscalls, file systems, loaders) on top of raw CPU emulation, enabling full binary execution without native hardware. - -## High-Level Overview - -``` -┌──────────────────────────────────────────────────┐ -│ User Script / qltool │ -├──────────────────────────────────────────────────┤ -│ Qiling Core (core.py) │ -│ hooks · state snapshots · patches │ -├────────────┬─────────────┬───────────────────────┤ -│ OS Layer │ Loader │ Memory Manager │ -│ (QlOs) │ (QlLoader) │ (QlMemoryManager) │ -│ syscalls │ ELF/PE/ │ map · read · write │ -│ APIs │ MachO/etc │ MMIO callbacks │ -├────────────┴─────────────┴───────────────────────┤ -│ Architecture Layer (QlArch) │ -│ registers · disassembly · calling conventions │ -├──────────────────────────────────────────────────┤ -│ Unicorn Engine (CPU) │ -│ instruction-level emulation │ -└──────────────────────────────────────────────────┘ -``` - -## Directory Structure - -``` -qiling/ -├── qiling/ # Core framework package -│ ├── core.py # Qiling class — main entry point and orchestrator -│ ├── core_hooks.py # Hook system (code, memory, interrupt, address hooks) -│ ├── core_hooks_types.py # Hook type definitions and dispatch -│ ├── core_struct.py # Endian-aware struct packing utilities -│ ├── const.py # Enumerations: QL_ARCH, QL_OS, QL_ENDIAN, etc. -│ ├── exception.py # Custom exception hierarchy -│ ├── utils.py # Component selection (select_arch, select_os, etc.) -│ ├── host.py # Host platform interface -│ ├── log.py # Logging configuration -│ │ -│ ├── arch/ # Architecture implementations -│ │ ├── arch.py # QlArch — abstract base class -│ │ ├── x86.py # x86 / x86-64 / 8086 -│ │ ├── arm.py # ARMv7 (32-bit) -│ │ ├── arm64.py # ARMv8 (64-bit) -│ │ ├── mips.py # MIPS32 -│ │ ├── riscv.py # RISC-V 32-bit -│ │ ├── riscv64.py # RISC-V 64-bit -│ │ ├── ppc.py # PowerPC 32-bit -│ │ ├── cortex_m.py # ARM Cortex-M (MCU) -│ │ ├── register.py # Register management -│ │ └── models.py # CPU model definitions -│ │ -│ ├── os/ # Operating system implementations -│ │ ├── os.py # QlOs — abstract base class -│ │ ├── memory.py # QlMemoryManager -│ │ ├── fcall.py # Function call interface (read params, set return) -│ │ ├── mapper.py # Syscall/API mapping -│ │ ├── path.py # Virtual filesystem path resolution -│ │ ├── filestruct.py # File descriptor abstraction -│ │ ├── thread.py # Threading primitives -│ │ ├── posix/ # POSIX shared layer (syscall handlers) -│ │ ├── linux/ # Linux-specific OS -│ │ ├── freebsd/ # FreeBSD-specific OS -│ │ ├── macos/ # macOS-specific OS -│ │ ├── qnx/ # QNX RTOS -│ │ ├── windows/ # Windows (Win32/Win64 API emulation) -│ │ ├── uefi/ # UEFI firmware services -│ │ ├── dos/ # DOS (8086 interrupts) -│ │ ├── mcu/ # Bare-metal microcontroller -│ │ └── blob/ # Raw binary blob execution -│ │ -│ ├── loader/ # Binary format loaders -│ │ ├── loader.py # QlLoader — abstract base class -│ │ ├── elf.py # ELF (Linux, FreeBSD, QNX) -│ │ ├── pe.py # PE (Windows) -│ │ ├── pe_uefi.py # PE for UEFI -│ │ ├── macho.py # Mach-O (macOS) -│ │ ├── dos.py # DOS COM/EXE -│ │ ├── mcu.py # MCU firmware images -│ │ └── blob.py # Raw binary blobs -│ │ -│ ├── cc/ # Calling conventions -│ │ ├── intel.py # cdecl, stdcall, ms64 -│ │ ├── arm.py # aarch32, aarch64 -│ │ ├── mips.py # MIPS o32 -│ │ ├── riscv.py # RISC-V ABI -│ │ └── ppc.py # PowerPC ABI -│ │ -│ ├── hw/ # Hardware peripheral emulation (MCU) -│ │ ├── peripheral.py # Base peripheral class -│ │ ├── hw.py # Hardware manager -│ │ ├── gpio/ # GPIO pins and interrupts -│ │ ├── timer/ # Timers, PWM, counters -│ │ ├── char/ # UART serial -│ │ ├── spi/ # SPI bus -│ │ ├── i2c/ # I2C bus -│ │ ├── net/ # Network interfaces -│ │ ├── analog/ # ADC/DAC -│ │ ├── intc/ # Interrupt controllers -│ │ ├── flash/ # Flash memory -│ │ ├── dma/ # DMA controllers -│ │ └── ... # Power, SD, misc peripherals -│ │ -│ ├── debugger/ # Debugger subsystem -│ │ ├── gdb/ # GDB remote protocol server -│ │ └── qdb/ # Qiling native debugger (with reverse debugging) -│ │ -│ ├── extensions/ # Optional extensions -│ │ ├── multitask.py # gevent-based multithreading -│ │ ├── trace.py # Instruction tracing -│ │ ├── coverage/ # Code coverage collection -│ │ ├── sanitizers/ # Memory sanitizers -│ │ ├── afl/ # AFL fuzzer integration -│ │ ├── r2/ # Radare2 integration -│ │ └── idaplugin/ # IDA Pro plugin -│ │ -│ └── profiles/ # Default OS configuration files (.ql) -│ ├── linux.ql # Stack/heap addresses, kernel params -│ ├── windows.ql -│ ├── macos.ql -│ └── ... -│ -├── qltool # CLI tool for running binaries -├── qltui.py # TUI interface -├── examples/ # Usage examples and sample scripts -├── tests/ # Test suite -└── docs/ # Documentation -``` - -## Core Components - -### `Qiling` (core.py) - -The central class. Every emulation session creates one `Qiling` instance that owns and wires together all other components: - -```python -ql = Qiling( - argv=["/path/to/binary", "arg1"], # binary + arguments - rootfs="/path/to/rootfs", # virtual filesystem root - ostype=QL_OS.LINUX, # target OS (auto-detected if omitted) - archtype=QL_ARCH.X8664, # target arch (auto-detected if omitted) -) -``` - -`Qiling` inherits from `QlCoreHooks` (hook management) and `QlCoreStructs` (endian-aware packing). Key properties: - -| Property | Type | Description | -|----------|------|-------------| -| `ql.arch` | `QlArch` | CPU architecture — registers, disassembly | -| `ql.os` | `QlOs` | Operating system — syscalls, I/O, APIs | -| `ql.loader` | `QlLoader` | Binary loader — parses and maps the executable | -| `ql.mem` | `QlMemoryManager` | Memory — map, read, write, MMIO | -| `ql.uc` | `unicorn.Uc` | Underlying Unicorn engine instance | - -### Architecture Layer (`qiling/arch/`) - -`QlArch` is the abstract base. Each architecture subclass configures: - -- **Unicorn engine** mode and architecture constants -- **Register** access via `ql.arch.regs` (read/write by name) -- **Disassembler** (Capstone) and **assembler** (Keystone) -- **Stack operations** — push, pop, pointer-width-aware -- **Endianness** and **bit width** (16/32/64) - -Supported: x86, x86-64, 8086, ARM, ARM64, MIPS, RISC-V (32/64), PowerPC, Cortex-M. - -### OS Layer (`qiling/os/`) - -`QlOs` is the abstract base. Each OS subclass provides: - -- **Syscall/interrupt dispatch** — routes CPU interrupts to handler functions -- **I/O streams** — `stdin`, `stdout`, `stderr` (interceptable) -- **Virtual filesystem** — path mapping through `rootfs` -- **Function call interface** (`ql.os.fcall`) — read params, set return values -- **API interception** — `set_api()` for hooking library functions - -**POSIX subsystem** (`os/posix/`): Shared syscall implementation for Linux, FreeBSD, macOS, and QNX. Individual syscall handlers live under `os/posix/syscall/`. - -**Windows** (`os/windows/`): Emulates Win32/Win64 API by hooking DLL imports. Includes registry, thread, handle, and fiber support. - -**UEFI** (`os/uefi/`): Emulates UEFI Boot Services, Runtime Services, and SMM. Uses a GUID database and protocol framework. - -### Loader Layer (`qiling/loader/`) - -`QlLoader` is the abstract base. Loaders parse a binary format, map segments into memory, resolve symbols, load dependencies, and set initial CPU state (PC, SP). - -| Loader | Format | Used By | -|--------|--------|---------| -| `QlLoaderELF` | ELF | Linux, FreeBSD, QNX | -| `QlLoaderPE` | PE/COFF | Windows, UEFI | -| `QlLoaderMacho` | Mach-O | macOS | -| `QlLoaderDOS` | COM/EXE | DOS | -| `QlLoaderMCU` | Firmware | Cortex-M MCU | -| `QlLoaderBlob` | Raw bytes | Shellcode / blob | - -### Memory Manager (`qiling/os/memory.py`) - -Wraps Unicorn's memory model with higher-level operations: - -- `map(addr, size, perms)` / `unmap(addr, size)` — region management -- `read(addr, size)` / `write(addr, data)` — data access -- `read_ptr(addr)` / `write_ptr(addr, val)` — pointer-width-aware access -- `read_cstring(addr)` — null-terminated string read -- MMIO callback support for memory-mapped peripherals - -### Calling Conventions (`qiling/cc/`) - -Each architecture has calling convention classes that abstract argument passing and return values. The `QlOs.fcall` interface uses these to provide a uniform way to read function parameters regardless of platform. - -## Execution Flow - -### 1. Initialization (`Qiling.__init__`) - -``` -Qiling(argv, rootfs) - │ - ├─ Detect arch/OS from binary headers (ql_guess_emu_env) - │ ELF magic → parse e_machine, OSABI - │ PE magic → parse Machine, Subsystem - │ MachO magic → parse CPU type - │ - ├─ Create QlArch (select_arch) → initializes Unicorn engine - ├─ Create QlLoader (select_loader) - ├─ Create QlMemoryManager - ├─ Create QlOs (select_os) - │ - └─ loader.run() - ├─ Parse binary format (headers, segments, sections) - ├─ Map segments into memory - ├─ Load shared libraries / DLLs - ├─ Setup stack, heap, TLS, auxiliary vectors - └─ Set initial PC (entry point) and SP -``` - -### 2. Execution (`ql.run()`) - -``` -ql.run(begin, end, timeout, count) - │ - ├─ Apply binary patches (ql.patch) - ├─ Write exit trap (guard address) - │ - └─ os.run() - └─ uc.emu_start(entry_point, exit_point) - │ - ├─ Unicorn executes instructions - │ - ├─ Hooks fire on: - │ ├─ Every instruction (hook_code) - │ ├─ Basic blocks (hook_block) - │ ├─ Memory access (hook_mem_read/write) - │ ├─ Interrupts (hook_intno) → syscall dispatch - │ ├─ Specific addresses (hook_address) - │ └─ Specific instructions (hook_insn) - │ - └─ Stops when PC reaches exit point, timeout, - ql.emu_stop(), or unhandled exception -``` - -### 3. Syscall Handling - -When the emulated binary issues a syscall (via `int 0x80`, `syscall`, `svc`, etc.): - -``` -CPU interrupt/instruction - → Unicorn interrupt hook - → QlOs syscall dispatcher - → Look up handler by syscall number - → Handler reads args via calling convention - → Emulates syscall behavior - → Sets return value in registers -``` - -## Component Selection - -Components are selected dynamically at runtime based on `QL_ARCH` and `QL_OS` enums. The `qiling/utils.py` module provides: - -- `select_arch(archtype)` → architecture class -- `select_os(ostype)` → OS class -- `select_loader(ostype)` → loader class -- `select_debugger(options)` → debugger class - -This makes it possible to support diverse platform combinations from a unified codebase. - -## Hook System - -The hook system (`core_hooks.py`) wraps Unicorn's callback mechanism: - -| Hook Type | Trigger | -|-----------|---------| -| `hook_code` | Every instruction (optionally within address range) | -| `hook_block` | Every basic block entry | -| `hook_address` | Specific address reached | -| `hook_intno` | CPU interrupt/exception | -| `hook_insn` | Specific instruction type (e.g., `syscall`) | -| `hook_mem_read` | Memory read | -| `hook_mem_write` | Memory write | -| `hook_mem_invalid` | Invalid memory access | - -Hooks can be scoped to address ranges and return `QL_HOOK_BLOCK` to suppress further hooks in the chain. - -## Key Extension Points - -- **Custom syscall handlers** — replace or extend any syscall -- **API hooking** — `ql.os.set_api(name, callback)` to intercept library calls -- **Binary patching** — `ql.patch(offset, data)` for runtime patching -- **State snapshots** — `ql.save()` / `ql.restore()` for checkpointing -- **Debugger attachment** — GDB remote protocol or native QDB debugger -- **Coverage/tracing** — `extensions/coverage/` and `extensions/trace.py` -- **Fuzzing** — AFL integration via `extensions/afl/` -- **Hardware peripherals** — register custom MCU peripherals in `hw/` - -## Dependencies - -| Package | Role | -|---------|------| -| `unicorn` (2.1.3) | CPU emulation engine | -| `capstone` | Disassembly | -| `keystone-engine` | Assembly | -| `pyelftools` | ELF parsing | -| `pefile` | PE parsing | -| `python-registry` | Windows registry emulation | -| `gevent` | Cooperative multithreading | -| `pyyaml` | Configuration parsing | - -Optional: `unicornafl` / `fuzzercorn` (fuzzing), `r2libr` (Radare2 integration). - -## Supported Platforms - -**Architectures:** x86, x86-64, 8086, ARM, ARM64, MIPS, RISC-V (32/64), PowerPC, Cortex-M - -**Operating Systems:** Linux, FreeBSD, macOS, Windows, UEFI, DOS, QNX, MCU (bare-metal), Blob - -## Improvement: Hybrid Kernel Architecture - -> Detailed implementation plan and task tracking: [TODO.md](TODO.md) - -### The Problem - -Qiling reimplements Linux kernel behavior syscall-by-syscall in Python. This works -for simple operations (file I/O, memory management, stat) but fundamentally cannot -scale to the full kernel surface: - -- **Networking**: No epoll. Sockets are proxied to host sockets with no isolation. - No real TCP state machine, no multicast, no raw/netlink sockets. -- **Multithreading**: Gevent greenlets are cooperative and single-threaded. No - preemption, no real concurrency. Futex is a gevent Event. Programs using pthreads, - mutexes, or condition variables don't behave correctly. -- **Signals**: `signal()`, `sigaction()`, `kill()` are mostly stubbed. No delivery, - no `EINTR`, no `SA_RESTART`. -- **Long tail**: capabilities, cgroups, namespaces, io_uring, seccomp, eBPF — the - kernel API surface is vast and growing. - -### The Solution - -A **hybrid architecture** that keeps Unicorn for CPU emulation and Qiling for -instrumentation, but offloads complex kernel subsystems to a real Linux kernel via -a **kernel proxy** helper process. Simple syscalls stay emulated in Python. - -``` -Syscall interrupt - → load_syscall() [UNCHANGED — existing dispatch in posix.py] - → check posix_syscall_hooks[CALL] - → proxy hook registered? → forward to kernel proxy - → no proxy hook? → existing Python handler [UNCHANGED] -``` - -The user explicitly chooses which missing syscalls to forward. Nothing is automatic — -by default Qiling behaves exactly as today. The integration uses the **existing -`set_syscall()` CALL hook mechanism** (`posix.py:128-143`), so `load_syscall()` and -all existing dispatch code remain completely unchanged. - -```python -proxy = KernelProxy(ql) -proxy.forward_syscall("epoll_create", returns_fd=True) -proxy.forward_syscall("epoll_ctl") -proxy.forward_syscall("epoll_wait") -ql.run() +# Qiling Framework — Architecture + +This is the control center for agent-readable architecture docs. Cross-cutting +facts live here; each subsystem is documented once in `ARCHITECTURE/.md` +(see the [Index](#index)). + +## Mission + +Qiling is an advanced binary emulation framework: it emulates and sandboxes +code in an isolated environment across multiple platforms and architectures. +Built on top of Unicorn Engine, it adds what raw CPU emulation lacks — +operating-system context (syscalls, APIs, filesystems, registries), executable +format loading, and dynamic linking (`README.md:13`). + +Supported combinations are defined authoritatively in code: + +- Architectures — `QL_ARCH` (`qiling/const.py:15`): 8086, x86, x86-64, ARM, + ARM64, Cortex-M, MIPS, RISC-V 32/64, PowerPC. +- Operating systems — `QL_OS` (`qiling/const.py:28`): Linux, FreeBSD, macOS, + Windows, UEFI, DOS, QNX, MCU (bare-metal), BLOB. +- Formats: ELF, PE, Mach-O, COM/MBR, Intel HEX / raw firmware. Kernel-module + emulation for Windows `.sys`, Linux `.ko`, macOS `.kext`. + +Headline capabilities: fine-grained instrumentation hooks (instruction, basic +block, memory access, interrupt, syscall/API), VM state save/restore, dynamic +hot patching, cross-architecture debugging (GDB server and the built-in Qdb +with reverse debugging), and fuzzing integration (AFL++/unicornafl). + +## Target Environment + +- **Shape**: pure-Python library (`from qiling import Qiling`), plus the + `qltool` CLI and `qltui.py` TUI. Also shipped as a Docker image + (`Dockerfile`) and PyPI package. +- **Python**: 3.10+ (`pyproject.toml:27`). Packaging via Poetry; version + 1.4.11-dev (`pyproject.toml:4`), status Beta, license GPLv2. +- **Core dependencies**: `unicorn ==2.1.3` (hard-pinned CPU emulator, + `pyproject.toml:29`), `capstone` (disassembly), `keystone-engine` + (assembly), `pefile`, `pyelftools`, `python-registry`, `gevent` + (multithread emulation), `pyyaml` (MCU profiles). Extras: `fuzz` → + unicornafl/fuzzercorn, `RE` → r2libr. +- **Hosts**: Linux, Windows, macOS. Some test suites are host-gated (Windows + PE tests need real system DLLs collected on Windows; see + [os-windows.md](ARCHITECTURE/os-windows.md)). +- **Fixtures**: `examples/rootfs/` is a git submodule + (https://github.com/qilingframework/rootfs.git) holding target binaries for + tests and examples. + +## Workspace Layout + +| Path | Holds | +| ---- | ----- | +| `qiling/` | The framework package (see Index for per-subsystem docs) | +| `qiling/profiles/` | Default per-OS config profiles (`linux.ql`, `windows.ql`, …) | +| `examples/` | Curated demo scripts, `fuzzing/`, `mcu/`, `shellcodes/`, `extensions/`, `scripts/` (DLL collectors), and the `rootfs/` submodule | +| `jexamples/` | Legacy example set (not covered by CI) | +| `tests/` | CI test suite — standalone `unittest` files run from `tests/` (`tests/test_onlinux.sh`, `tests/test_pe.bat`, `tests/test_macho.sh`) | +| `docs/` | Stubs and assets; real documentation lives at https://docs.qiling.io | +| `qltool` | CLI entry point (see [cli.md](ARCHITECTURE/cli.md)) | +| `qltui.py` | Terminal UI invoked via `qltool qltui` | +| `Dockerfile` | Multi-stage Poetry wheel build on `python:3-slim` | +| `pyproject.toml` / `poetry.lock` | Packaging and dependency pins | +| `.github/workflows/` | CI (`build-ci.yml`: Windows + Ubuntu × Python 3.11/3.13, plus a Docker container job), Docker publish, PyPI publish, Gitee mirror sync | + +## Boot / Entry Flow + +From CLI to emulated instructions (details live in the module docs): + +1. `qltool` parses args and builds kwargs; `run` and `code` subcommands end in + `Qiling(**ql_args)` (`qltool:276`) — see [cli.md](ARCHITECTURE/cli.md). +2. `Qiling.__init__` (`qiling/core.py:35`) is the composition root. Order: + guess arch/OS from the binary if not given → instantiate arch → init + struct/hook mixins → logger → profile → loader → memory manager → OS layer + → hardware manager (bare-metal only) → `loader.run()` maps the target into + emulated memory (`qiling/core.py:154-195`). All name→class resolution is in + `qiling/utils.py` — see [core.md](ARCHITECTURE/core.md). +3. `Qiling.run()` (`qiling/core.py:561`) attaches the debugger if configured, + applies queued patches, writes the exit trap, then delegates to + `os.run()` — see [debugger.md](ARCHITECTURE/debugger.md) and the OS docs. +4. The OS layer drives `Qiling.emu_start` (`qiling/core.py:743`), the thin + wrapper over Unicorn's `uc.emu_start`. Syscalls/APIs raised by the emulated + code are dispatched back into the OS layer + ([os-posix.md](ARCHITECTURE/os-posix.md), + [os-windows.md](ARCHITECTURE/os-windows.md), + [os-baremetal.md](ARCHITECTURE/os-baremetal.md)). + +## Roadmap + +Maturity-based — Qiling is a released project in maintenance/beta +(v1.4.11-dev). There are no in-repo milestones; module Status is `done` when +its test suite proves it. Feature requests and the forward-looking wishlist are tracked in +GitHub issue [#333](https://github.com/qilingframework/qiling/issues/333) +(the `TODO` file is a pointer to it). Known cross-cutting gaps: + +- `ChangeLog` stops at 1.4.6 (`ChangeLog:4`) while `pyproject.toml:4` says + 1.4.11-dev. +- macOS is dropped from the CI matrix and the macOS/kext job is commented out + (`.github/workflows/build-ci.yml:12`, `:84`). +- `jexamples/` is legacy and unexercised by CI. + +## Development Loop + +Coding Discipline governs writing; Review Checks govern review. This +loop connects them and defines when work is ready to release. + +```text +Frame → Write → Prove → Review → Gate + ▲ findings │ + └────────────────────────┘ ``` -Under the hood, `forward_syscall()` registers a CALL hook that serializes the -arguments and sends them to a helper process (the kernel proxy) which executes -the real syscall and returns the result. For syscalls that return FDs, the result -is wrapped in a `ql_proxy_fd` object and stored in Qiling's FD table. Since the -FD table is already polymorphic (`ql_socket`, `ql_file`, `ql_pipe`), existing -handlers like `ql_syscall_read` and `ql_syscall_close` dispatch through the proxy -FD's `.read()`/`.close()` methods automatically — no changes needed. - -### Phases - -| Phase | Scope | Risk | Goal | -|-------|-------|------|------| -| 0 | Proof of concept | Low | User manually forwards specific syscalls — zero existing code changed | -| 1 | Networking foundation | Low-Med | Specific hooks for socket syscalls, `ql_proxy_fd`, TCP works | -| 2 | Complete networking | Medium | epoll, poll/select, network namespaces | -| 3 | Real threading | **High** | One Unicorn per thread, shared memory, real futex | -| 4 | Signals | Medium | Real signal delivery, EINTR, handler execution | -| 5 | Integration | Low | API polish, fallback, platform support, benchmarks | - -Phase 0 gives users explicit control — they identify which missing syscalls to -forward and the proxy handles them. Phases 1-2 add pointer-aware forwarding for -networking with pre-built forwarders so users don't have to wire up each syscall. -Phase 3 (threading) is the highest-risk change and is deferred until networking is -stable. Each phase preserves backward compatibility — hybrid mode is opt-in, default -behavior is unchanged. - -### Alternatives Considered - -- **Run a real kernel in Unicorn**: Unicorn doesn't emulate hardware (interrupt - controllers, MMU page tables, timers). Would require rebuilding QEMU system mode. -- **ptrace-based execution**: Run natively, intercept syscalls. Fast, but no - cross-architecture support and limited instruction-level hooks. -- **User-Mode Linux (UML)**: Run the kernel as a userspace process. x86-only, - somewhat unmaintained, complex syscall bridge. -- **Auto-forward all unimplemented syscalls**: Forward every missing syscall - automatically. Convenient but unpredictable — hard to debug, may forward syscalls - that shouldn't be (security, state leaks). Explicit user control is safer. - -The hybrid approach was chosen because it preserves Qiling's core value -(instrumentation + cross-arch emulation) while getting real kernel behavior where -it matters most — without modifying the existing dispatch path. - -## Testing - -Tests live in `tests/` and are organized by platform: `test_elf.py`, `test_pe.py`, `test_macho.py`, `test_dos.py`, `test_mcu.py`, etc. They use binaries from `examples/rootfs/` as test fixtures. +### The loop + +**1. Frame.** Convert the request into a goal with an observable check. +Inspect the request, code, docs, and repository conventions; record the +narrowest supported assumptions. Ask one focused question only when a +required decision cannot be discovered or safely inferred and guessing +would materially change the result. Once framed, continue without an +approval pause. + +**2. Write.** Make the smallest change that reaches the goal. Add no +unrequested features or abstractions, match local style, touch only +in-scope code, and remove only orphans created by the change. + +**3. Prove.** Run relevant tests and retain observable evidence. + +*Survey the suite before touching it.* Before adding, changing, merging, +or deleting any test, inventory the whole suite: enumerate every test +file and case name, then read in full each test whose subject, fixtures, +or assertions touch this change. Use a subagent for broad inventory when +supported. From that inventory decide the complete set of test edits at +once — what to change, what to add, what to merge, what to remove — each +backed by `file:line`, then execute only that plan. Never write a test +before the survey, and never discover existing coverage afterward. + +The plan obeys four rules: + +- **Reuse or extend first.** Add a case to the test that already owns + the behavior or shares its setup, fixtures, and subject. A new test + function or file is justified only when the survey found no existing + test owning the behavior, or when merging would hide which case + failed. +- **Add only what the goal needs.** A bug fix needs a reproducing + regression test; a new capability needs a test of its claimed + behavior. Nothing further. +- **Retire what this change made obsolete.** Delete tests whose behavior + no longer exists, and merge tests this change turned into duplicates, + citing the surviving test. Leave unrelated pre-existing tests alone; + record suspected redundancy under **Open Gaps / Roadmap**. +- **Never delete to reach green.** A failing test is a finding for + Write. Removal requires evidence that its behavior is gone or is still + covered elsewhere, cited by `file:line`. + +Coverage of claimed behavior must not decrease. A failure returns +directly to Write, never forward to Review. + +**4. Review.** Walk all seven Review Checks as separate passes. Read +whole affected files, not only the diff. Every finding needs `file:line` +evidence. Use an independent agent or isolated pass for Fit, +Dependencies, and Security when available. + +**5. Gate.** Apply the Definition of Done. Any unticked criterion, +`blocker`, or unresolved `major` returns its evidence to Write. All +criteria passing means the change is ready for public or production +release. There is no separate approval or reporting phase. + +### Definition of Done + +**Correctness** + +- The framed goal and its named check pass. +- Tests cover claimed behavior and pass; a bug fix has a regression test. +- The suite was surveyed before any test was written, changed, or + deleted; no added test duplicates coverage another test owns, and no + removal left claimed behavior uncovered. +- The owning module's **How to Test** command passes with evidence. +- The project builds and tests from a fresh clone without local-only + dependencies. + +**Review** + +- All seven Review Checks ran; none was skipped or assumed. +- No `blocker` or unresolved `major` remains. +- Nits were applied or consciously declined. + +**Legibility and contract** + +- A new maintainer can build, test, run, and understand public behavior + from the docs. +- Every changed line serves the goal; no drive-by formatting, debugging + remnants, commented-out code, secrets, tokens, or local paths remain. +- Public names, signatures, errors, and recovery are intelligible. +- Architecture docs and `file:line` references are current. +- Breaking changes, deprecations, dependencies, licenses, and attribution + are handled; commit or PR text explains why. + +### Iterating without thrashing + +- Every pass closes a named finding and touches only what it names. +- Nits alone do not trigger another pass. +- Re-run Prove after every fix. +- Two no-change passes force Gate re-evaluation: release if Done passes; + otherwise return the surviving evidence to Frame. +- Three passes on one finding return automatically to Frame for a new + approach. +- Never widen scope to satisfy a finding. Record out-of-scope work under + **Open Gaps / Roadmap**. + +## Coding Discipline + +### 1. Think Before Coding + +- Understand the request, code, goal, and repository conventions first. +- Record assumptions and choose the narrowest evidence-backed reading. +- Prefer the simpler approach when it reaches the same verified goal. +- Ask only during planning and only for a required answer that cannot be + discovered or safely inferred. + +### 2. Simplicity First + +- Implement only what was requested. +- Do not add single-use abstractions, speculative flexibility, or checks + for impossible conditions. +- If the implementation is materially larger than the problem, simplify + it. + +### 3. Surgical Changes + +- Do not refactor, reformat, or clean up unrelated code. +- Match the surrounding style. +- Remove imports, variables, and functions made unused by this change; + leave pre-existing dead code alone unless requested. +- Every changed line must trace to the stated goal. + +### 4. Goal-Driven Execution + +Turn work into verifiable outcomes, then loop until they pass: + +- Add validation → invalid inputs are rejected by a named passing test. +- Fix a bug → a regression test fails before the fix and passes after. +- Refactor → behavior tests pass before and after. + +Give every plan step its own check. Strengthen vague criteria from +repository evidence before implementation. + +### Project-Specific Deviations + +- Emulation fidelity beats abstraction: syscall, API, and peripheral + implementations mirror the real platform's observable behavior even + when that means repetitive per-OS or per-chip code. Cross-OS + "unification" is a scope increase, not a simplification. +- Coverage is demand-driven by design (see the OS and HW module docs). + Adding an unrequested syscall, Win32 API, or peripheral register is + out of scope; record it under the owning module's **Open Gaps / + Roadmap**. +- `unicorn` is hard-pinned (`pyproject.toml:29`). Changing it, or any + behavior that depends on its version, is a project-wide event and + never an incidental part of another change. + +## Review Checks + +Run every check against every change before merge. Keep checks separate. + +Four rules bind all checks: + +- **Evidence or no finding.** Every finding cites `file:line`. +- **The repository is authoritative.** Demand only conventions visible + in the tree. +- **Read files, not only hunks.** Context can invalidate a finding or + reveal unreachable code, unused parameters, and hidden duplication. +- **Review the change, never the author.** Describe code and impact, not + how or by whom it was produced. + +### 1. Style + +Check indentation and local file conventions. Mixed indentation is +`major`; a consistent new file using the wrong local indent is `nit`. +Leave machine-checkable formatting to existing formatters and linters; +never demand unrelated reformatting. + +### 2. Naming + +Compare new names with nearby precedents before filing a finding. If the +repository is inconsistent, demand nothing. A local mismatch is `nit`; +an inconsistent public name is `major`. + +### 3. Duplication + +Search distinctive constants, errors, fields, and call sequences—not +only symbol names—for code performing the same job. Cite both sites and +the remedy. Cross-layer duplication is `major`; small local repetition +is `nit`. Similar code with meaningfully different branches is not +duplication. + +### 4. Quality + +Require followable control flow, errors handled where they occur, and +abstractions proportional to the problem. Swallowed errors, +inappropriate prints, unexplained magic values, and dead branches are +`major`. Remove unrequested configurability, one-caller wrappers, filler +comments, debugging remnants, and unrelated formatting. Missing tests +belong to Prove, not this check. + +### 5. Fit + +Read `ARCHITECTURE.md` and the owning module doc before the diff. Check +scope, layering, ownership, public-API growth, and performance claims. A +layering violation or unjustified public API is `major`. Architectural or +public-behavior changes must update the relevant docs in the same change. + +### 6. Dependencies + +Check manifests and imports, maintenance, supply-chain risk, advisories, +install-time behavior, license, transitive cost, and whether the standard +library is sufficient. An unjustified top-level dependency is `major`; +a live advisory or abandoned upstream is `blocker`. Incomplete evidence +does not pass. + +### 7. Security + +Check both defects and widened exposure: unsafe memory access, unchecked +sizes or offsets, integer overflow, path traversal, unsafe +deserialization, command construction, committed secrets, and unbounded +untrusted input. Trace input to impact; without a reachable path there is +no finding. A real defect is `major`; a trust-boundary break is `blocker`. +Describe the fix without publishing exploit steps. + +### Project-Specific Deviations + +- **Security, scope.** Qiling *emulates* untrusted binaries; guest code + doing something hostile inside the sandbox is the product working, not + a finding. Findings target the host boundary: rootfs escape via path + handling (`qiling/os/path.py`), unchecked guest-controlled sizes or + offsets reaching host allocations or `struct` unpacking, and parser + input in `qiling/loader/` reachable from an untrusted image. +- **Dependencies.** `pyproject.toml` is the only manifest. A new + top-level runtime dependency is `blocker` absent an explicit request; + optional integrations belong in an extra (`fuzz`, `RE`). + +### Severity and the merge threshold + +| Severity | Effect | +| -------- | ------ | +| `blocker` | Must not merge. | +| `major` | Must be resolved before merge. | +| `nit` | Apply or consciously decline. | +| `info` | Context or a question; no action implied. | + +Merge only with no `blocker` and no unresolved `major`. A check that did +not run does not pass. Findings feed Write and Gate directly; they do not +create a reporting phase. + +## Index + +- [core.md](ARCHITECTURE/core.md) — the `Qiling` facade, hook engine, component selection, profiles, logging, exceptions +- [arch.md](ARCHITECTURE/arch.md) — CPU layer: registers, Unicorn instance, disassembler, calling conventions +- [loader.md](ARCHITECTURE/loader.md) — binary format loaders: ELF, PE, PE/UEFI, Mach-O, DOS, MCU firmware, raw blobs +- [os-base.md](ARCHITECTURE/os-base.md) — shared OS layer: `QlOs`, memory manager/heap, fcall marshalling, fs mapper, path virtualization +- [os-posix.md](ARCHITECTURE/os-posix.md) — POSIX syscall emulation: Linux, FreeBSD, macOS, QNX +- [os-windows.md](ARCHITECTURE/os-windows.md) — Windows API emulation, UEFI boot/runtime/SMM services, DOS interrupts +- [os-baremetal.md](ARCHITECTURE/os-baremetal.md) — MCU and raw-blob execution modes, cooperative multitasking +- [hw.md](ARCHITECTURE/hw.md) — MMIO peripheral emulation for bare-metal targets, board/chip definitions +- [debugger.md](ARCHITECTURE/debugger.md) — GDB remote-serial server and the built-in Qdb debugger +- [extensions.md](ARCHITECTURE/extensions.md) — AFL fuzzing, coverage/tracing, sanitizers, r2/IDA integration, pipes, reports +- [cli.md](ARCHITECTURE/cli.md) — `qltool` CLI and `qltui.py` TUI diff --git a/ARCHITECTURE/arch.md b/ARCHITECTURE/arch.md new file mode 100644 index 000000000..75e9d3b08 --- /dev/null +++ b/ARCHITECTURE/arch.md @@ -0,0 +1,58 @@ +# Arch — CPU architecture layer + +## Goal + +Own everything CPU-specific: the Unicorn `Uc` instance, register access, stack +primitives, disassembler/assembler, CPU models, and per-arch calling +conventions. This is the bottom layer — everything else reads `ql.arch`; arch +depends only on Unicorn/Capstone/Keystone. Mature released infrastructure; +maturity-based status. + +## Status + +`done` — all architectures exercised by the CI suite; CPU model selection +covered by `tests/test_cpu_models.py`. + +## Code Structure + +| File | Role | +| ---- | ---- | +| `qiling/arch/arch.py` | Abstract base `QlArch`: owns `uc`, `regs`, stack push/pop, save/restore, disassembler | +| `qiling/arch/x86.py` | `QlArchIntel` base + `QlArchA8086`/`QlArchX86`/`QlArchX8664`, GDT/MSR wiring | +| `qiling/arch/arm.py`, `arm64.py` | ARM/AArch64, thumb handling, coprocessor access | +| `qiling/arch/cortex_m.py` | Cortex-M on top of ARM: NVIC-style interrupt entry/exit for MCU mode | +| `qiling/arch/mips.py`, `riscv.py`, `riscv64.py`, `ppc.py` | Remaining architectures | +| `qiling/arch/register.py` | `QlRegisterManager` — attribute-style register read/write | +| `qiling/arch/models.py` | CPU model enums (`X86_CPU_MODEL` … `RISCV64_CPU_MODEL`) | +| `qiling/arch/msr.py`, `cpr.py`, `cpr64.py` | x86 MSRs, ARM/ARM64 coprocessor registers | +| `qiling/arch/utils.py` | `QlArchUtils`: disassembly output for verbose/trace modes | +| `qiling/cc/__init__.py` + `intel.py`, `arm.py`, `mips.py`, `ppc.py`, `riscv.py` | Calling conventions (arg/retval marshalling) consumed by `qiling/os/fcall.py` | + +## Key Types and Entry Points + +- `qiling/arch/arch.py:22` - `QlArch(ABC)` - cached properties `uc` (`:34`), `regs` (`:42`), `stack_push/stack_pop` (`:52`/`:66`), `save/restore` via UcContext (`:108`/`:112`), `disassembler` (`:117`). +- `qiling/arch/register.py:11` - `QlRegisterManager` - `ql.arch.regs.rax`-style access, backed by per-arch `*_const.py` tables. +- `qiling/arch/x86.py:22,53,79,111` - `QlArchIntel` / `QlArchA8086` / `QlArchX86` / `QlArchX8664`. +- `qiling/arch/cortex_m.py:67` - `QlArchCORTEX_M(QlArchARM)` - plus `QlInterruptContext` (`:25`) for exception entry/exit in MCU mode. +- `qiling/arch/models.py` - CPU model enums selected via the `cputype` kwarg (resolved in `select_arch`, `qiling/utils.py:376`). +- `qiling/cc/__init__.py:9` - `QlCC` - abstract calling convention; `QlCommonBaseCC` (`:110`); e.g. `qiling/cc/intel.py` defines `cdecl`/`stdcall`/`ms64`/`macosx64`. + +## Interactions + +- Instantiated first by [core.md](core.md) (`qiling/core.py:154`); `Qiling.uc` proxies `arch.uc` (`qiling/core.py:479`). +- [loader.md](loader.md) and the OS layers use `arch.regs` and stack primitives to set up entry state. +- `qiling/cc/` is consumed by `QlFunctionCall` in [os-base.md](os-base.md) for API argument marshalling. +- [debugger.md](debugger.md) reads/writes registers through this layer. + +## How to Test + +```sh +cd tests && python3 test_cpu_models.py # pass = unittest "OK", exit code 0 +``` + +- Broader arch coverage comes for free from `test_shellcode.py` (5 archs) and the per-OS suites. + +## Open Gaps / Roadmap + +- PowerPC and RISC-V have fewer OS-level tests than x86/ARM/MIPS (no dedicated POSIX suite beyond `tests/test_riscv.py`). +- Thumb state handling has a known fixup in `Qiling.emu_start` (`qiling/core.py:743`) rather than in the arch layer itself. diff --git a/ARCHITECTURE/cli.md b/ARCHITECTURE/cli.md new file mode 100644 index 000000000..40b13a98c --- /dev/null +++ b/ARCHITECTURE/cli.md @@ -0,0 +1,48 @@ +# CLI — qltool and qltui + +## Goal + +Give users a no-code way to run emulations: `qltool run` executes a binary +against a rootfs, `qltool code` runs shellcode (hex/asm/bin), `qltool +examples` prints usage samples, and `qltool qltui` launches an interactive +TUI that gathers the same options. Mature released infrastructure; +maturity-based status. + +## Status + +`done` — covered by `tests/test_qltool.py`, which shells out to `qltool` for +run/code subcommands including gdb attach and coverage output. + +## Code Structure + +| File | Role | +| ---- | ---- | +| `qltool` | Executable argparse CLI; builds kwargs and drives `Qiling` | +| `qltui.py` | questionary/pyfx/termcolor TUI; collects options, returned to qltool | + +## Key Types and Entry Points + +- `qltool:189` - `run()` - argparse setup with subcommands `run`, `code`, `examples`, `qltui`; enum-mapping actions translate `--arch/--os/--endian/--verbose` strings to `QL_ARCH`/`QL_OS` enums (`qltool:59-75`). +- `qltool:129` - `handle_run(options)` - builds `{'argv': [file]+args, 'rootfs': ...}`. +- `qltool:78` - `handle_code(options)` - reads hex/asm/bin shellcode, assembling asm via `qiling.arch.utils.assembler` (`qltool:104`). +- `qltool:276` - `ql = Qiling(**ql_args)` - the single construction point; then optional Qdb (`:279`), gdbserver (`:285`), coverage-wrapped `ql.run()` (`:306-310`), JSON report (`:312`), exit with `ql.os.exit_code` (`:321`). + +## Interactions + +- Thin client of [core.md](core.md): constructs `Qiling` and calls `run()`. +- Attaches [debugger.md](debugger.md) via `--gdb host:port` / `--qdb [--rr]`. +- Uses [extensions.md](extensions.md) for `--coverage-file` (drcov) and `--json` report output. +- `qltool examples` mirrors scripts documented in `examples/README.md`. + +## How to Test + +```sh +cd tests && python3 test_qltool.py # pass = unittest "OK", exit 0 +``` + +- Manual smoke test: `./qltool run -f examples/rootfs/x8664_linux/bin/x8664_hello --rootfs examples/rootfs/x8664_linux` — pass = prints `Hello, World!`. + +## Open Gaps / Roadmap + +- `qltui.py` (TUI) has no automated tests — interactive only. +- `qltool` predates subcommand-style config files; complex setups (fs mappers, custom hooks) still require the Python API. diff --git a/ARCHITECTURE/core.md b/ARCHITECTURE/core.md new file mode 100644 index 000000000..4b16504e2 --- /dev/null +++ b/ARCHITECTURE/core.md @@ -0,0 +1,61 @@ +# Core — the Qiling facade and plumbing + +## Goal + +Own the public API and object lifecycle of an emulation: the `Qiling` class +composes arch, loader, memory, OS, and (for bare-metal) hardware components, +and exposes hooks, memory/register access, patching, and save/restore to +users. No roadmap milestone applies — this is mature released infrastructure; +status is maturity-based per the control center. + +## Status + +`done` — exercised by the whole test suite; the How to Test command below +boots shellcode end-to-end through `Qiling.__init__` → `run()` → `emu_start`. + +## Code Structure + +| File | Role | +| ---- | ---- | +| `qiling/core.py` | `Qiling` class: constructor/composition root, `run`, `emu_start/stop`, `save/restore`, `patch`, properties | +| `qiling/core_hooks.py` | `QlCoreHooks` mixin: wraps Unicorn hooks, dispatches to `Hook` lists | +| `qiling/core_hooks_types.py` | `Hook`, `HookAddr`, `HookIntr`, `HookRet` records | +| `qiling/core_struct.py` | `QlCoreStructs` mixin: endian/bit-width-aware pack/unpack helpers | +| `qiling/utils.py` | Name→class resolution: `select_arch/loader/os/component/debugger`, binary format sniffing, profile loading | +| `qiling/const.py` | Enums: `QL_ARCH`, `QL_OS`, `QL_VERBOSE`, `QL_INTERCEPT`, `QL_STATE`, groupings `QL_OS_POSIX`/`QL_OS_BAREMETAL` | +| `qiling/exception.py` | `QlErrorBase` and ~20 subclasses (`QlErrorArch`, `QlSyscallError`, …) | +| `qiling/host.py` | `QlHost`: describes the *hosting* platform (for pass-through features) | +| `qiling/log.py` | Logger setup, colored/plain formatters, regex filtering behind `Qiling.filter` | +| `qiling/profiles/*.ql` | Default per-OS config (memory layout, kernel uid/gid/pid) merged with user overrides | + +## Key Types and Entry Points + +- `qiling/core.py:35` - `Qiling(QlCoreHooks, QlCoreStructs)` - the facade; `__init__` composes components in fixed order: arch (`:154`) → mixins (`:157`) → logger → profile (`:178`) → loader (`:183`) → memory (`:188`) → OS (`:189`) → hw if bare-metal (`:191`) → `loader.run()` (`:195`). +- `qiling/core.py:561` - `Qiling.run(begin, end, timeout, count)` - attaches debugger, applies patches, writes exit trap, delegates to `os.run()`. +- `qiling/core.py:743` - `Qiling.emu_start(begin, end, timeout, count)` - thin wrapper over `uc.emu_start`; manages thumb bit, `QL_STATE`, exception re-raise. +- `qiling/core.py:609` / `:658` - `save()` / `restore()` - snapshot regs/mem/fd/os per-component. +- `qiling/core_hooks.py:150` - `QlCoreHooks` - hook registration API: `hook_code` (`:400`), `hook_block` (`:422`), `hook_address` (`:550`), `hook_intno` (`:575`), `hook_mem_read/write` (`:592`/`:610`), `hook_insn` (`:646`), `hook_del` (`:686`). +- `qiling/utils.py:278` - `ql_guess_emu_env(path)` - sniffs arch/OS/endian from pathname, ELF, Mach-O, or PE headers when not given. +- `qiling/utils.py:297,376,409,323,332` - `select_loader/arch/os/component/debugger` - dynamic-import factories; core never imports concrete subclasses. +- `qiling/utils.py:419` - `profile_setup(ostype, user_config)` - YAML for MCU, else ConfigParser over `qiling/profiles/.ql` + user overrides. + +## Interactions + +- Instantiates every other subsystem: [arch.md](arch.md), [loader.md](loader.md), [os-base.md](os-base.md) (memory + OS), [hw.md](hw.md) (bare-metal only). +- Lazily instantiates [debugger.md](debugger.md) inside `run()` via `select_debugger`. +- [extensions.md](extensions.md) and [cli.md](cli.md) consume only this public API. +- Hook dispatch protocol (`QL_HOOK_BLOCK`, `qiling/const.py:77`) is honored by the OS layers when they intercept syscalls/APIs. + +## How to Test + +```sh +cd tests && python3 test_shellcode.py # pass = unittest "OK", exit code 0 +``` + +- Exercises `Qiling(code=...)` construction and `run()` across x86/x86-64/ARM/ARM64/MIPS shellcode for Linux and Windows. + +## Open Gaps / Roadmap + +- `ChangeLog` lags the released version (1.4.6 vs 1.4.8 in `pyproject.toml`). +- `unicorn` is hard-pinned to 2.1.3; upgrading Unicorn is a project-wide event. +- Feature wishlist lives in GitHub issue [#333](https://github.com/qilingframework/qiling/issues/333). diff --git a/ARCHITECTURE/debugger.md b/ARCHITECTURE/debugger.md new file mode 100644 index 000000000..52156ec9c --- /dev/null +++ b/ARCHITECTURE/debugger.md @@ -0,0 +1,56 @@ +# Debugger — GDB server and Qdb + +## Goal + +Let users debug emulated targets: a GDB remote-serial-protocol server so any +GDB/IDA/lldb front end can attach cross-architecture, and Qdb — a built-in +interactive CLI debugger with stepping, branch prediction, and record/replay +reverse debugging. Mature released infrastructure; maturity-based status. + +## Status + +`done` — Qdb covered by `tests/test_qdb.py`; the GDB server by +`tests/test_debugger.py` (spawns a real client session). + +## Code Structure + +| File | Role | +| ---- | ---- | +| `qiling/debugger/debugger.py` | Base `QlDebugger` | +| `qiling/debugger/gdb/gdb.py` | `QlGdb`: GDB remote-serial-protocol server, plus the `GdbSerialConn` transport | +| `qiling/debugger/gdb/utils.py` | `QlGdbUtils`: breakpoint table and the per-instruction `dbg_hook` that services breakpoints, stepping, and async interrupts | +| `qiling/debugger/gdb/xmlregs.py`, `gdb/xml/` | Target-description XML per arch for modern GDB clients | +| `qiling/debugger/qdb/qdb.py` | `QlQdb`: interactive Cmd-based debugger | +| `qiling/debugger/qdb/arch/` | Per-arch Qdb support (arm, intel, mips) | +| `qiling/debugger/qdb/branch_predictor/` | Predicts branch targets for step/next | +| `qiling/debugger/qdb/render/` | Register/stack/disasm view rendering | + +## Key Types and Entry Points + +- `qiling/debugger/debugger.py:13` - `QlDebugger` - base; `run()` starts the session. +- `qiling/debugger/gdb/gdb.py:84` - `QlGdb(QlDebugger)` - listens on ip:port, translates RSP packets to Qiling hook/mem/reg operations; `run()` (`:139`) serves the session. +- `qiling/debugger/gdb/gdb.py:817` - `GdbSerialConn` - the socket transport; `poll_interrupt()` (`:856`) is a non-blocking check for a client `\x03`, wired into `QlGdbUtils.check_interrupt` (`qiling/debugger/gdb/gdb.py:145`) so a running guest can be broken into asynchronously. +- `qiling/debugger/gdb/utils.py:16` - `QlGdbUtils` - `dbg_hook` (`:48`) runs per instruction to service breakpoints, single-step, and interrupts; `bp_insert`/`bp_remove` (`:81`/`:94`); `resume_emu` (`:107`). +- Stop replies report `SIGTRAP` (`qiling/debugger/gdb/gdb.py:49`) for both single-step (`:243`) and async-interrupt stops (`:262`). +- `qiling/debugger/qdb/qdb.py:59` - `QlQdb(Cmd, QlDebugger)` - CLI loop; `rr` mode enables record/replay reverse debugging. +- Activation: set `ql.debugger = True | "gdb" | "gdb:0.0.0.0:9999" | "qdb" | "qdb:rr"` (`qiling/core.py:437`); instantiated lazily in `Qiling.run` via `select_debugger` (`qiling/utils.py:332`). +- `qltool` flags: `--gdb` and `--qdb` (see [cli.md](cli.md)). + +## Interactions + +- Instantiated by [core.md](core.md) at `Qiling.run` time, not construction. +- Reads/writes state exclusively through public APIs: registers via [arch.md](arch.md), memory via [os-base.md](os-base.md), breakpoints via `hook_address` ([core.md](core.md)). +- The IDA plugin in [extensions.md](extensions.md) offers an alternative front end over the same public API. + +## How to Test + +```sh +cd tests && python3 test_qdb.py # pass = unittest "OK", exit 0 +``` + +- GDB server: `cd tests && python3 test_debugger.py` — starts `QlGdb` and drives a scripted client. + +## Open Gaps / Roadmap + +- Qdb per-arch support covers arm/cortex-m/mips/intel; RISC-V and PPC lack Qdb arch modules (`qiling/debugger/qdb/arch/`). +- Record/replay (`qdb:rr`) stores full state per step — memory-heavy on long runs. diff --git a/ARCHITECTURE/extensions.md b/ARCHITECTURE/extensions.md new file mode 100644 index 000000000..a9f0e7387 --- /dev/null +++ b/ARCHITECTURE/extensions.md @@ -0,0 +1,61 @@ +# Extensions — optional tooling on top of the core + +## Goal + +House everything that builds on the public `Qiling` API without being required +by it: fuzzing (AFL), code-coverage and execution-trace writers, a heap +sanitizer, radare2 and IDA Pro integration, fake stdio pipes, and JSON run +reports. Mature released infrastructure; maturity-based status. + +## Status + +`done` — history/coverage tracker tested in CI; AFL and r2 integrations need +optional extras (`fuzz`, `RE`); the IDA plugin needs IDA Pro (untested in CI). + +## Code Structure + +| File | Role | +| ---- | ---- | +| `qiling/extensions/afl/` | `ql_afl_fuzz` (unicornafl bridge) + `QlFuzzer` harness base class | +| `qiling/extensions/coverage/` | `collect_coverage` context manager; writers in `formats/` (drcov, drcov_exact, ezcov, history) | +| `qiling/extensions/tracing/` | Tenet-style execution trace writers (`formats/`) | +| `qiling/extensions/trace.py` | Disassembly tracing: full trace or ring-buffer history | +| `qiling/extensions/sanitizers/heap.py` | Canary-based heap sanitizer (UAF/OOB detection) | +| `qiling/extensions/r2/r2.py` | radare2 (r2libr) integration: sections/symbols/functions/xrefs of the loaded target | +| `qiling/extensions/idaplugin/qilingida.py` | IDA Pro plugin driving Qiling emulation from IDA | +| `qiling/extensions/pipe.py` | Fake stdio streams for hijacking emulated I/O (fuzzing staple) | +| `qiling/extensions/report/report.py` | `generate_report(ql)`: JSON summary of a run | +| `qiling/extensions/multitask.py` | Cooperative-multitask Unicorn wrapper (documented in [os-baremetal.md](os-baremetal.md)) | +| `qiling/extensions/mcu/` | Board/chip definitions (documented in [hw.md](hw.md)) | +| `qiling/extensions/winsdkapi.py` | Windows API signature decorator glue | + +## Key Types and Entry Points + +- `qiling/extensions/afl/afl.py:21` / `:87` - `ql_afl_fuzz` / `ql_afl_fuzz_custom` - hand control to AFL++ via unicornafl; harness base `QlFuzzer` (`qiling/extensions/afl/qlfuzzer.py:14`). +- `qiling/extensions/coverage/utils.py:48` - `collect_coverage(ql, name, file)` - context manager writing e.g. drcov files (used by `qltool --coverage-file`). +- `qiling/extensions/trace.py:145` / `:180` - `enable_full_trace` / `enable_history_trace` - disassembly tracing via `hook_code`. +- `qiling/extensions/sanitizers/heap.py:16` - `QlSanitizedMemoryHeap` - drop-in replacement for `ql.os.heap` with canaries and free-list checks. +- `qiling/extensions/r2/r2.py:135` - `R2(ql)` - rzpipe-backed analysis of the loaded image. +- `qiling/extensions/pipe.py` - `SimpleInStream`/`SimpleOutStream` (`:62`/`:69`) - assigned to `ql.os.stdin`/`stdout`. +- `qiling/extensions/report/report.py:56` - `generate_report(ql)` - JSON report (used by `qltool --json`). + +## Interactions + +- Everything here consumes only the public API of [core.md](core.md) (hooks, mem, regs) and [os-base.md](os-base.md) (heap, stdio, syscall/API overrides). +- The heap sanitizer wraps `QlMemoryHeap` from [os-base.md](os-base.md) (demo: `examples/uefi_sanitized_heap.py` with [os-windows.md](os-windows.md) UEFI). +- Fuzzing harnesses live in `examples/fuzzing/` and pair AFL with `pipe.py` and `set_syscall` from [os-posix.md](os-posix.md). +- `qltool` wires in coverage and report generation ([cli.md](cli.md)). + +## How to Test + +```sh +cd tests && python3 test_history.py # pass = unittest "OK", exit 0 +``` + +- r2 integration (needs `pip install qiling[RE]`): `cd tests && python3 test_r2.py`. +- AFL (needs `pip install qiling[fuzz]` + AFL++): run a harness from `examples/fuzzing/linux_x8664/`. + +## Open Gaps / Roadmap + +- `afl/` and `r2/` depend on optional extras not installed by default; the IDA plugin cannot be CI-tested (requires an IDA Pro license). +- Two overlapping trace mechanisms exist (`trace.py` and `tracing/`); no unified interface. diff --git a/ARCHITECTURE/hw.md b/ARCHITECTURE/hw.md new file mode 100644 index 000000000..c1466aaaf --- /dev/null +++ b/ARCHITECTURE/hw.md @@ -0,0 +1,53 @@ +# HW — peripheral emulation for bare-metal targets + +## Goal + +Emulate memory-mapped peripherals (GPIO, UART/char, SPI, I2C, timers, DMA, +interrupt controllers, flash, …) so MCU firmware runs against realistic +hardware. Peripherals are instantiated from a YAML profile naming which +peripheral class sits at which MMIO base address. Only active in bare-metal +(MCU) mode. Mature released infrastructure; maturity-based status. + +## Status + +`done` — exercised by `tests/test_mcu.py` against STM32F1/F4 and GD32VF1 +firmware images (UART echo, freertos, blink, crc, dma_clock, i2c/spi/lcd). + +## Code Structure + +| File | Role | +| ---- | ---- | +| `qiling/hw/hw.py` | `QlHwManager`: creates peripherals from profile, maps MMIO, steps them each tick | +| `qiling/hw/peripheral.py` | Base `QlPeripheral` (+ `QlPeripheralUtils`) | +| `qiling/hw/analog/ char/ dma/ flash/ gpio/ i2c/ intc/ math/ mem/ misc/ net/ power/ sd/ spi/ timer/` | One directory per peripheral class, chip-specific implementations inside | +| `qiling/hw/const/` | Register layout constants per chip family | +| `qiling/hw/connectivity.py`, `external_device/` | External-interface plumbing (e.g. connecting a fake device to a bus) | +| `qiling/extensions/mcu/` | Board/chip definitions (stm32f4xx, gd32vf1, nxp, atmel, bes): memory maps naming peripheral class + base address, passed as `env=` to `Qiling` | + +## Key Types and Entry Points + +- `qiling/hw/hw.py:60` - `QlHwManager` - available as `ql.hw`; created by core only when `ql.baremetal` (`qiling/core.py:191`). +- `qiling/hw/hw.py:67` - `QlHwManager.create(label, struct, base)` - instantiates a peripheral from the profile entry and maps its MMIO region. +- `qiling/hw/hw.py:135` - `QlHwManager.step()` - advances every peripheral one tick; called from the MCU run loop. +- `qiling/hw/peripheral.py:132` - `QlPeripheral(QlPeripheralUtils)` - base class: a ctypes register struct + read/write handlers on the MMIO region. +- `qiling/extensions/mcu/stm32f4/stm32f407.py` (and siblings) - chip `env` dicts consumed at `Qiling(..., env=...)` construction. + +## Interactions + +- Created by [core.md](core.md) for bare-metal targets; driven by the MCU run loop in [os-baremetal.md](os-baremetal.md) (`QlHwManager.step()` between execution slices). +- Interrupt controller peripherals (`intc/`) raise exceptions delivered through `QlInterruptContext` in [arch.md](arch.md). +- MMIO regions are mapped through `QlMemoryManager` ([os-base.md](os-base.md)). +- Profile parsing (YAML) is in `profile_setup` ([core.md](core.md), `qiling/utils.py:419`). + +## How to Test + +```sh +cd tests && python3 test_mcu.py # pass = unittest "OK", exit 0 +``` + +- Exercises GPIO, UART, EXTI, I2C, SPI, DMA, CRC, RTC, watchdog peripherals on real firmware from `examples/rootfs/mcu/`. + +## Open Gaps / Roadmap + +- Peripheral fidelity is demand-driven: registers behave as observed firmware needs, not per full datasheets. +- Chip coverage limited to families under `qiling/extensions/mcu/`; adding a chip means writing its memory map + any missing peripheral classes. diff --git a/ARCHITECTURE/loader.md b/ARCHITECTURE/loader.md new file mode 100644 index 000000000..da4b71966 --- /dev/null +++ b/ARCHITECTURE/loader.md @@ -0,0 +1,58 @@ +# Loader — binary format loaders + +## Goal + +Map the target binary into emulated memory and prepare execution state: parse +the format, load segments/sections and dependencies (ld.so, DLLs, dylibs), +build stack/argv/auxv or OS structures (PEB/TEB, UEFI protocols), and set +entry/exit points. One loader per format, selected by OS type. Mature released +infrastructure; maturity-based status. + +## Status + +`done` — every format loader is exercised by its OS test suite; the How to +Test command proves the ELF path end-to-end. + +## Code Structure + +| File | Role | +| ---- | ---- | +| `qiling/loader/loader.py` | Base `QlLoader` + `Image` record; abstract `run()`, save/restore | +| `qiling/loader/elf.py` | ELF for Linux/FreeBSD/QNX: segments, interpreter (ld.so), stack, auxv | +| `qiling/loader/pe.py` | Windows PE: DLL resolution + `QlPeCache`, PEB/TEB/LDR via `Process` | +| `qiling/loader/pe_uefi.py` | UEFI DXE/SMM module loading, protocol installation | +| `qiling/loader/macho.py` | Mach-O (with its own `macho_parser/` package) | +| `qiling/loader/dos.py` | DOS COM/EXE/MBR | +| `qiling/loader/mcu.py` | MCU firmware: Intel HEX (`IhexParser`), ELF, or raw bin per YAML profile | +| `qiling/loader/blob.py` | Raw blobs loaded at a fixed address (e.g. u-boot) | + +## Key Types and Entry Points + +- `qiling/loader/loader.py:21` - `QlLoader` - base; `Image` NamedTuple (`:15`), abstract `run()` (`:62`), `skip_exit_check` (`:27`). +- `qiling/loader/elf.py:68` - `QlLoaderELF` - loads binary + interpreter, builds stack/auxv (`AUXV` enum `:32`), sets `entry_point`/`elf_entry`. +- `qiling/loader/pe.py:817` - `QlLoaderPE(QlLoader, Process)` - `Process` (`:77`) builds PEB/TEB/LDR; `QlPeCache` (`:49`) caches parsed DLLs behind the `libcache` kwarg (wired in `select_loader`, `qiling/utils.py:300`). +- `qiling/loader/pe_uefi.py:26` - `QlLoaderPE_UEFI` - loads DXE/SMM modules and installs protocols into the UEFI context. +- `qiling/loader/macho.py:70` - `QlLoaderMACHO`. +- `qiling/loader/mcu.py:57` - `QlLoaderMCU` - with `IhexParser` (`:15`). +- OS→loader mapping: `select_loader` (`qiling/utils.py:297`) — LINUX/FREEBSD/QNX→elf, MACOS→macho, WINDOWS→pe, UEFI→pe_uefi, DOS→dos, MCU→mcu, BLOB→blob. + +## Interactions + +- Instantiated by [core.md](core.md) (`qiling/core.py:183`); `loader.run()` fires at the end of `Qiling.__init__` (`qiling/core.py:195`). +- Uses `ql.mem` ([os-base.md](os-base.md)) for mapping and `ql.arch` ([arch.md](arch.md)) for initial register/stack state. +- The PE loader builds Windows process structures with [os-windows.md](os-windows.md) components (heap, handles); the UEFI loader populates the context in `qiling/os/uefi/`. +- IAT entries recorded here drive address-based API hooking in [os-windows.md](os-windows.md). +- The MCU loader reads the YAML profile owned by [core.md](core.md) and memory maps defined via [hw.md](hw.md). + +## How to Test + +```sh +cd tests && python3 -m unittest test_elf.ELFTest.test_elf_linux_x8664 # pass = "OK", exit 0 +``` + +- Full loader coverage: `test_elf.py`, `test_pe.py` (Windows host), `test_macho.py` (macOS host), `test_dos.py`, `test_mcu.py`, `test_blob.py`, `test_uefi.py`. + +## Open Gaps / Roadmap + +- PE and Mach-O loading depend on host-collected libraries (`examples/scripts/dllscollector.bat` / `dylibcollector.sh`); not runnable from a Linux checkout alone. +- `QlPeCache` cache invalidation is manual (delete cache files) — no versioning. diff --git a/ARCHITECTURE/os-baremetal.md b/ARCHITECTURE/os-baremetal.md new file mode 100644 index 000000000..0b7a68a7e --- /dev/null +++ b/ARCHITECTURE/os-baremetal.md @@ -0,0 +1,61 @@ +# OS bare-metal — MCU firmware and raw blobs + +## Goal + +Execute code with no operating system. MCU mode runs microcontroller firmware +(STM32, GD32V, NXP, …) with peripheral emulation and interrupt scheduling on +a cooperative multitasking wrapper around Unicorn; BLOB mode runs raw binaries +(e.g. u-boot) at a fixed load address with no OS services at all. Mature +released infrastructure; maturity-based status. + +## Status + +`done` — MCU covered by `tests/test_mcu.py` (STM32F1/F4, GD32VF1 firmware), +BLOB by `test_blob.BlobTest.test_uboot_arm` and `tests/test_edl.py`. The other +case in `tests/test_blob.py` is blocked on a missing fixture (see Open Gaps). + +## Code Structure + +| File | Role | +| ---- | ---- | +| `qiling/os/mcu/mcu.py` | `QlOsMcu` run loop + `MCUTask` driving Unicorn and peripheral steps | +| `qiling/os/blob/blob.py` | `QlOsBlob`: minimal run loop for raw binaries | +| `qiling/extensions/multitask.py` | `UnicornTask` / `MultiTaskUnicorn`: cooperative multitasking over Unicorn | + +## Key Types and Entry Points + +- `qiling/os/mcu/mcu.py:41` - `QlOsMcu(QlOs)` - run loop; steps hardware between execution chunks and delivers interrupts. +- `qiling/os/mcu/mcu.py:17` - `MCUTask(UnicornTask)` - the firmware execution task. +- `qiling/extensions/multitask.py:26` / `:152` - `UnicornTask` / `MultiTaskUnicorn(Uc)` - task-switching Unicorn subclass MCU mode runs on. +- `qiling/os/blob/blob.py:14` - `QlOsBlob(QlOs)` - runs `entry_point` → `exit_point` with no OS services. +- MCU selection: `QL_OS.MCU` is the only member of `QL_OS_BAREMETAL` (`qiling/const.py:74`); `ql.baremetal` (`qiling/core.py:357`) gates hardware-manager creation. + +## Interactions + +- Both subclass [os-base.md](os-base.md) `QlOs`. +- MCU mode drives [hw.md](hw.md): `QlHwManager.step()` is called between execution slices; board/chip memory maps come from `qiling/extensions/mcu/` passed as `env=`. +- Interrupt entry/exit uses `QlInterruptContext` in [arch.md](arch.md) (`qiling/arch/cortex_m.py:25`). +- Firmware images are loaded by `QlLoaderMCU` / `QlLoaderBLOB` ([loader.md](loader.md)); MCU profiles are YAML (`profile_setup`, [core.md](core.md)). +- Fuzzing MCU firmware (e.g. `examples/fuzzing/stm32f429/`) combines this mode with [extensions.md](extensions.md) AFL support. + +## How to Test + +```sh +cd tests && python3 -m unittest test_blob.BlobTest.test_uboot_arm # pass = "OK", exit 0 +``` + +- MCU (also proves [hw.md](hw.md)): `cd tests && python3 test_mcu.py` — pass = `Ran 18 tests … OK`. +- Qualcomm EDL loader: `cd tests && python3 test_edl.py` — pass = `Ran 1 test … OK`. +- The whole `test_blob.py` file does **not** pass from a clean checkout; see + Open Gaps. + +## Open Gaps / Roadmap + +- Supported chip families are those with board definitions in `qiling/extensions/mcu/` (STM32F1/F4, GD32VF1, NXP, Atmel, BES); new chips need new peripheral maps. +- BLOB mode provides no services by design — targets needing hardware must use MCU mode instead. +- `test_blob.BlobTest.test_blob_raw` (`tests/test_blob.py:85`) errors with + `FileNotFoundError` on a clean checkout: it reads + `examples/rootfs/blob/example_raw.bin` (`tests/test_blob.py:96`), but the + pinned `examples/rootfs` submodule ships only `u-boot.bin.img`. Either the + fixture must be added upstream and the submodule bumped, or the test skipped + when the fixture is absent. diff --git a/ARCHITECTURE/os-base.md b/ARCHITECTURE/os-base.md new file mode 100644 index 000000000..756a0e044 --- /dev/null +++ b/ARCHITECTURE/os-base.md @@ -0,0 +1,60 @@ +# OS base — shared OS-layer services + +## Goal + +Own the OS-agnostic services every OS personality builds on: the abstract +`QlOs` class, the emulated memory manager and heap, function-call argument +marshalling (via calling conventions), filesystem virtualization (rootfs +confinement + host mapping), file descriptor objects, green threads, and +API-call statistics. Mature released infrastructure; maturity-based status. + +## Status + +`done` — path virtualization and struct helpers have dedicated unit tests; +everything else is exercised by every OS suite. + +## Code Structure + +| File | Role | +| ---- | ---- | +| `qiling/os/os.py` | Abstract `QlOs`: utils/stats/path/fs-mapper composition, `set_api`, user-defined API dict, abstract `run()` | +| `qiling/os/memory.py` | `QlMemoryManager` (map/unmap/search, `ql.mem`) and `QlMemoryHeap` | +| `qiling/os/fcall.py` | `QlFunctionCall`: read args / write retval using `qiling/cc/` conventions | +| `qiling/os/mapper.py` | `QlFsMapper`/`QlFsMappedObject`: redirect emulated paths to host files/objects | +| `qiling/os/path.py` | `QlOsPath`: rootfs confinement and virtual→host path conversion | +| `qiling/os/filestruct.py`, `disk.py` | File-descriptor and raw-disk objects | +| `qiling/os/thread.py` | `QlThread(Greenlet)` base for multithread emulation | +| `qiling/os/stats.py` | `QlOsStats`: API/syscall call statistics | +| `qiling/os/struct.py`, `utils.py`, `const.py` | ctypes-based struct helpers, OS utils, shared constants | + +## Key Types and Entry Points + +- `qiling/os/os.py:24` - `QlOs` - composes utils/stats/path/fs-mapper; `user_defined_api` keyed by `QL_INTERCEPT`; abstract `run()` (`:240`). +- `qiling/os/os.py:225` - `QlOs.set_api(target, handler, intercept)` - user override of an emulated API (address- or name-based). +- `qiling/os/memory.py:40` - `QlMemoryManager` - `ql.mem`; instantiated by core *before* the OS (`qiling/core.py:188`). +- `qiling/os/memory.py:678` - `QlMemoryHeap` - heap used by Windows/UEFI APIs and the sanitizers extension. +- `qiling/os/fcall.py:21` - `QlFunctionCall` - argument/return marshalling on top of `qiling/cc/`. +- `qiling/os/mapper.py:64` - `QlFsMapper` - behind `ql.add_fs_mapper` (`qiling/core.py:701`). +- `qiling/os/path.py:16` - `QlOsPath` - rootfs-confined path resolution. +- `qiling/os/thread.py:11` - `QlThread(Greenlet)` - gevent-based thread base. + +## Interactions + +- Instantiated by [core.md](core.md): memory at `qiling/core.py:188`, the concrete OS at `:189` (via `select_os`). +- Uses [arch.md](arch.md) calling conventions (`qiling/cc/`) inside `QlFunctionCall`. +- Subclassed by [os-posix.md](os-posix.md) (`QlOsPosix`), [os-windows.md](os-windows.md) (`QlOsWindows`, `QlOsUefi`, `QlOsDos`), and [os-baremetal.md](os-baremetal.md) (`QlOsMcu`, `QlOsBlob`). +- [loader.md](loader.md) maps segments through `QlMemoryManager`. +- The heap is wrapped by `QlSanitizedMemoryHeap` in [extensions.md](extensions.md). + +## How to Test + +```sh +cd tests && python3 test_pathutils.py && python3 test_struct.py # pass = both print "OK", exit 0 +``` + +- `test_pathutils.py` proves rootfs path virtualization (incl. case-insensitive Windows semantics); `test_struct.py` proves the struct helpers. + +## Open Gaps / Roadmap + +- Multithread emulation (gevent) is opt-in via `multithread=True` and less battle-tested than single-threaded mode (see `tests/test_elf_multithread.py`). +- `QlOs.run()` contract is loose: each personality implements its own loop; no shared scheduler outside POSIX threads and MCU multitask. diff --git a/ARCHITECTURE/os-posix.md b/ARCHITECTURE/os-posix.md new file mode 100644 index 000000000..8c8dd84c7 --- /dev/null +++ b/ARCHITECTURE/os-posix.md @@ -0,0 +1,58 @@ +# OS POSIX — Linux, FreeBSD, macOS, QNX + +## Goal + +Emulate POSIX operating systems by intercepting syscalls: a shared +`QlOsPosix` layer owns the per-arch syscall ABI, number→name mapping, and the +file-descriptor table; per-OS subclasses add their personality (Linux futex +and procfs, macOS mach ports, QNX message passing). Mature released +infrastructure; maturity-based status. + +## Status + +`done` — Linux is the flagship target with the largest suite; FreeBSD/QNX +covered by dedicated tests; macOS tests are host-gated (see Open Gaps). + +## Code Structure + +| File | Role | +| ---- | ---- | +| `qiling/os/posix/posix.py` | `QlOsPosix`: syscall ABI per arch, dispatcher, `set_syscall`, fd table | +| `qiling/os/posix/syscall/` | Syscall implementations, one module per family (unistd, mman, socket, fcntl, signal, ioctl, …) | +| `qiling/os/posix/shm.py`, `msq.py` | SysV IPC (shared memory, message queues) | +| `qiling/os/linux/linux.py` | `QlOsLinux`: syscall hook wiring and the dynamic-ELF run loop | +| `qiling/os/linux/` | `futex.py`, `thread.py` (green threads), `procfs.py`, `map_syscall.py`/`syscall_nums.py`, `kernel_api/` (for .ko emulation) | +| `qiling/os/freebsd/freebsd.py` | `QlOsFreebsd` | +| `qiling/os/macos/macos.py` | `QlOsMacos` + mach ports/tasks/kext support | +| `qiling/os/qnx/qnx.py` | `QlOsQnx` + QNX message passing (`message.py`) | + +## Key Types and Entry Points + +- `qiling/os/posix/posix.py:48` - `QlOsPosix(QlOs)` - per-arch syscall ABI tables (`:69`), fd table, SysV IPC. +- `qiling/os/posix/posix.py:170` - `load_syscall()` - the dispatcher: syscall id → name (via per-OS `map_syscall`) → handler in `os/posix/syscall/*` or user override. +- `qiling/os/posix/posix.py:132` - `set_syscall(target, handler, intercept)` - user syscall hijack (by number or name, CALL/ENTER/EXIT). +- `qiling/os/posix/posix.py:100` - `__get_syscall_mapper(archtype)` - loads `qiling/os//map_syscall.py` dynamically. +- `qiling/os/linux/linux.py:26` - `QlOsLinux(QlOsPosix)` - `hook_syscall` (`:137`) registered on interrupt/insn hooks. +- `qiling/os/linux/linux.py:148` - `QlOsLinux.run()` - shellcode single-shot, gevent thread manager when `multithread=True`, or the common path: run ld.so to `elf_entry`, apply lib patches, then run to exit. + +## Interactions + +- Subclasses [os-base.md](os-base.md) `QlOs`; fd objects come from `qiling/os/filestruct.py`. +- Syscall entry is an arch hook: interrupt/insn hooks from [core.md](core.md) (`QlCoreHooks`) fire `hook_syscall`, using [arch.md](arch.md) registers per the syscall ABI. +- Runs binaries prepared by [loader.md](loader.md) (`QlLoaderELF`, `QlLoaderMACHO`). +- Path/fs access goes through `QlOsPath`/`QlFsMapper` in [os-base.md](os-base.md). +- Fuzzing harnesses in [extensions.md](extensions.md) commonly hijack fd 0 with `pipe.py` and syscalls via `set_syscall`. + +## How to Test + +```sh +cd tests && python3 test_posix.py # pass = unittest "OK", exit 0 +``` + +- Full POSIX coverage in CI: `tests/test_onlinux.sh` (adds `test_elf.py` targets, `test_elf_multithread.py`, `test_elf_ko.py`, `test_qnx.py`, `test_android.py`). + +## Open Gaps / Roadmap + +- macOS emulation tests (`test_macho.py`, `test_macho_kext.py`) need a macOS host with collected dylibs; the macOS CI job is commented out in `.github/workflows/build-ci.yml`. +- `test_elf_ko.py` requires a one-time `unzip -P infected m0hamed_rootkit.ko.zip` in `examples/rootfs/x86_linux/kernel/`. +- Syscall coverage is demand-driven — unimplemented syscalls raise/log and get added as targets need them. diff --git a/ARCHITECTURE/os-windows.md b/ARCHITECTURE/os-windows.md new file mode 100644 index 000000000..cf4187796 --- /dev/null +++ b/ARCHITECTURE/os-windows.md @@ -0,0 +1,59 @@ +# OS Windows family — Windows, UEFI, DOS + +## Goal + +Emulate API-call-driven (rather than syscall-driven) environments. Windows: +Win32/NT API emulation with handles, registry, and PE process structures. +UEFI: boot/runtime/SMM services and protocol database for DXE/SMM modules. +DOS: BIOS/DOS interrupt services for COM/EXE/MBR targets. Mature released +infrastructure; maturity-based status. + +## Status + +`done` — Windows PE suite runs in CI on a Windows host; UEFI and DOS suites +run on Linux. Windows tests are platform-gated locally (need collected DLLs). + +## Code Structure + +| File | Role | +| ---- | ---- | +| `qiling/os/windows/windows.py` | `QlOsWindows`: component setup (handles, registry, clipboard, fiber), run loop | +| `qiling/os/windows/dlls/` | Win32 API implementations per DLL (kernel32/, ntdll.py, msvcrt.py, advapi32.py, …) | +| `qiling/os/windows/fncc.py`, `api.py` | Decorators declaring API signatures/calling conventions | +| `qiling/os/windows/handle.py`, `registry.py`, `clipboard.py`, `fiber.py`, `thread.py`, `structs.py` | Windows kernel-object emulation | +| `qiling/os/uefi/uefi.py` | `QlOsUefi` run loop | +| `qiling/os/uefi/bs.py`, `rt.py`, `ds.py`, `smm.py`, `protocols/` | Boot/runtime/DXE/SMM services and protocol implementations | +| `qiling/os/uefi/context.py` | UEFI execution contexts populated by the loader | +| `qiling/os/dos/dos.py` | `QlOsDos` run loop | +| `qiling/os/dos/interrupts/` | INT 10h/13h/15h/16h/21h… service handlers | + +## Key Types and Entry Points + +- `qiling/os/windows/windows.py:33` - `QlOsWindows(QlOs)` - `__setup_components` (`:156`) builds handle manager/registry/clipboard/fiber; `run()` (`:201`). +- `qiling/os/windows/fncc.py` - `@winsdkapi` decorator - declares an API's calling convention and typed params; implementations live in `dlls/`. +- API dispatch is address-based: IAT addresses recorded by the PE loader are hooked and marshalled through `QlFunctionCall` — user overrides via `QlOs.set_api` (`qiling/os/os.py:225`). +- `qiling/os/uefi/uefi.py:22` - `QlOsUefi(QlOs)` - executes DXE/SMM modules; services in `bs.py`/`rt.py`/`smm.py` are installed as callable tables. +- `qiling/os/dos/dos.py:33` - `QlOsDos(QlOs)` - dispatches BIOS/DOS interrupts from `interrupts/`. + +## Interactions + +- All three subclass [os-base.md](os-base.md) `QlOs` and allocate from `QlMemoryHeap`. +- [loader.md](loader.md): `QlLoaderPE` builds PEB/TEB and records IAT hook addresses; `QlLoaderPE_UEFI` installs protocols into `qiling/os/uefi/context.py`; `QlLoaderDOS` sets real-mode state. +- Argument marshalling uses `QlFunctionCall` + `qiling/cc/intel.py` conventions ([arch.md](arch.md)). +- The registry emulation reads hive files from the rootfs via [os-base.md](os-base.md) path services. +- `examples/uefi_sanitized_heap.py` pairs UEFI with the heap sanitizer from [extensions.md](extensions.md). + +## How to Test + +```sh +cd tests && python3 test_uefi.py # pass = unittest "OK", exit 0 (Linux-runnable) +``` + +- Windows host only: `cd tests && cmd.exe /C test_pe.bat` after running `examples/scripts/dllscollector.bat` (CI does this in `.github/workflows/build-ci.yml`). +- DOS: `cd tests && python3 test_dos.py` (uses scripted keyboard input). + +## Open Gaps / Roadmap + +- Windows tests require a Windows host: system DLLs/registry hives are not redistributable and must be collected locally. +- Win32 API surface is demand-driven; unimplemented APIs raise `QlErrorSyscallNotFound`-style errors and are added as samples need them. +- SMM emulation covers common protocols only (see `qiling/os/uefi/smm.py`). diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 000000000..708f0d2e3 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +ARCHITECTURE.md \ No newline at end of file diff --git a/examples/rootfs b/examples/rootfs index df3fa4dfc..f71f45fe1 160000 --- a/examples/rootfs +++ b/examples/rootfs @@ -1 +1 @@ -Subproject commit df3fa4dfc0b9d4164f8678699d8923df847eb3d2 +Subproject commit f71f45fe1a39d58d8b8cae717f55cebeb37f63c7 diff --git a/pyproject.toml b/pyproject.toml index efaa4734a..afa35a8d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,13 @@ [tool.poetry] urls = { homepage = "https://qiling.io" } name = "qiling" -version = "1.4.10" +version = "1.4.11-dev" description = "Qiling is an advanced binary emulation framework that cross-platform-architecture" authors = ["KaiJern Lau (xwings) "] license = "GPLv2" readme = "README.md" classifiers = [ - "Development Status :: 5 - Production/Stable", + "Development Status :: 3 - Beta", "Intended Audience :: Developers", "Topic :: Software Development :: Disassemblers", "Topic :: System :: Emulators", diff --git a/qiling/debugger/gdb/gdb.py b/qiling/debugger/gdb/gdb.py index f6d6498d8..77c0623fa 100644 --- a/qiling/debugger/gdb/gdb.py +++ b/qiling/debugger/gdb/gdb.py @@ -15,6 +15,7 @@ import os import socket +import select import re import tempfile from functools import partial @@ -30,7 +31,7 @@ ) from qiling import Qiling -from qiling.const import QL_ARCH, QL_ENDIAN, QL_OS, QL_STATE +from qiling.const import QL_ARCH, QL_ENDIAN, QL_OS from qiling.debugger import QlDebugger from qiling.debugger.gdb.xmlregs import QlGdbFeatures from qiling.debugger.gdb.utils import QlGdbUtils @@ -56,6 +57,21 @@ SIGCONT = 17 SIGSTOP = 18 +# translate a unicorn cpu fault into the closest posix signal, shared by the +# continue ('c') and single-step ('s') handlers when emulation faults +UC_ERROR_SIGMAP = { + UC_ERR_READ_UNMAPPED : SIGSEGV, + UC_ERR_WRITE_UNMAPPED : SIGSEGV, + UC_ERR_FETCH_UNMAPPED : SIGSEGV, + UC_ERR_WRITE_PROT : SIGSEGV, + UC_ERR_READ_PROT : SIGSEGV, + UC_ERR_FETCH_PROT : SIGSEGV, + UC_ERR_READ_UNALIGNED : SIGBUS, + UC_ERR_WRITE_UNALIGNED : SIGBUS, + UC_ERR_FETCH_UNALIGNED : SIGBUS, + UC_ERR_INSN_INVALID : SIGILL +} + # common replies REPLY_ACK = b'+' REPLY_EMPTY = b'' @@ -124,6 +140,10 @@ def run(self): server = GdbSerialConn(self.ip, self.port, self.ql.log) killed = False + # let the run hook break into a free-running guest when the client sends + # an async interrupt (ctrl-c / \x03); see QlGdbUtils.dbg_hook. + self.gdb.check_interrupt = server.poll_interrupt + def __hexstr(value: int, nibbles: int = 0) -> str: """Encode a value into a hex string. """ @@ -226,33 +246,24 @@ def handle_c(subcmd: str) -> Reply: try: self.gdb.resume_emu() except UcError as err: - sigmap = { - UC_ERR_READ_UNMAPPED : SIGSEGV, - UC_ERR_WRITE_UNMAPPED : SIGSEGV, - UC_ERR_FETCH_UNMAPPED : SIGSEGV, - UC_ERR_WRITE_PROT : SIGSEGV, - UC_ERR_READ_PROT : SIGSEGV, - UC_ERR_FETCH_PROT : SIGSEGV, - UC_ERR_READ_UNALIGNED : SIGBUS, - UC_ERR_WRITE_UNALIGNED : SIGBUS, - UC_ERR_FETCH_UNALIGNED : SIGBUS, - UC_ERR_INSN_INVALID : SIGILL - } - # determine signal from uc error; default to SIGTERM - reply = f'S{sigmap.get(err.errno, SIGTERM):02x}' + reply = f'S{UC_ERROR_SIGMAP.get(err.errno, SIGTERM):02x}' except KeyboardInterrupt: # emulation was interrupted with ctrl+c reply = f'S{SIGINT:02x}' else: - if getattr(self.ql.arch, 'effective_pc', self.ql.arch.regs.arch_pc) == self.gdb.last_bp: + if self.gdb.interrupted: + # emulation was stopped by an async interrupt from the client + reply = f'S{SIGINT:02x}' + elif getattr(self.ql.arch, 'effective_pc', self.ql.arch.regs.arch_pc) == self.gdb.last_bp: # emulation stopped because it hit a breakpoint reply = f'S{SIGTRAP:02x}' else: - # emulation has completed successfully - reply = f'W{self.ql.os.exit_code:02x}' + # emulation has completed successfully. note bare-metal os layers + # do not have an exit code (issue #1276) + reply = f'W{getattr(self.ql.os, "exit_code", 0):02x}' return reply @@ -662,10 +673,17 @@ def handle_v(subcmd: str) -> Reply: for grp in groups: cmd, *tid = grp.split(':', maxsplit=1) - if cmd in ('c', f'C{SIGTRAP:02x}'): + # 'C sig' and 'S sig' resume or step while delivering a signal + # to the guest. we do not deliver signals, so the signal value + # is ignored and the action is carried out as a plain resume or + # step. matching only 'C05' and 'S05' here made clients that + # resume with any other pending signal (e.g. 'S0f' after a stop + # reply we sent) receive an empty reply and bail out with + # 'Invalid remote reply' (issue #1377) + if cmd[:1] in ('c', 'C'): return handle_c('') - elif cmd in ('s', f'S{SIGTRAP:02x}'): + elif cmd[:1] in ('s', 'S'): return handle_s('') # FIXME: not sure how to handle multiple command @@ -678,11 +696,23 @@ def handle_s(subcmd: str) -> Reply: """Perform a single step. """ - self.gdb.resume_emu(steps=1) + try: + self.gdb.resume_emu(steps=1) + except UcError as err: + # stepping faulted; report the closest posix signal + return f'S{UC_ERROR_SIGMAP.get(err.errno, SIGTERM):02x}' + + except KeyboardInterrupt: + # emulation was interrupted with ctrl+c + return f'S{SIGINT:02x}' - # if emulation has been stopped, signal program termination - if self.ql.emu_state is QL_STATE.STOPPED: - return f'S{SIGTERM:02x}' + # emu_start always leaves emu_state as STOPPED after a step, so that + # cannot tell an ordinary step apart from the guest exiting (see + # issues #1377 and #1538). instead, the guest has terminated only + # when the step carried pc all the way to the emulation exit point. + if getattr(self.ql.arch, 'effective_pc', self.ql.arch.regs.arch_pc) == self.gdb.exit_point: + # program terminated; report its exit code + return f'W{getattr(self.ql.os, "exit_code", 0):02x}' # otherwise, this is just single stepping return f'S{SIGTRAP:02x}' @@ -823,6 +853,27 @@ def close(self): self.client.close() self.sock.close() + def poll_interrupt(self) -> bool: + """Non-blocking check for an async interrupt from the client. + + While the target is running the only thing gdb sends is a bare ``\\x03`` + break byte (it waits for a stop reply before sending anything else), so + any readable bytes here are that interrupt or a stray protocol ack. + Returns True if a break was seen. Called from the run hook, so it must + never block. + """ + + readable, _, _ = select.select([self.client], [], [], 0) + if not readable: + return False + + try: + incoming = self.client.recv(self.BUFSIZE) + except (ConnectionError, OSError): + return False + + return b'\x03' in incoming + def readpackets(self) -> Iterator[bytes]: """Iterate through incoming packets in an active connection until it is terminated. diff --git a/qiling/debugger/gdb/utils.py b/qiling/debugger/gdb/utils.py index 5e3b208bf..6413d1eed 100644 --- a/qiling/debugger/gdb/utils.py +++ b/qiling/debugger/gdb/utils.py @@ -14,6 +14,11 @@ class QlGdbUtils: + # how often (in guest instructions) the run hook polls the client socket for + # an async interrupt. small enough to feel instant, large enough that the + # extra non-blocking socket check does not dominate the per-instruction hook. + INTR_POLL_INTERVAL = 200 + def __init__(self, ql: Qiling, entry_point: int, exit_point: int): self.ql = ql @@ -21,6 +26,14 @@ def __init__(self, ql: Qiling, entry_point: int, exit_point: int): self.swbp = set() self.last_bp = None + # async-interrupt support: `check_interrupt` is a callable installed by the + # gdb stub that returns True when the client sent a break (ctrl-c / \x03) + # while the target was running. `interrupted` records that the last resume + # stopped for that reason (rather than a breakpoint or normal exit). + self.check_interrupt = None + self.interrupted = False + self._poll_counter = 0 + def __entry_point_hook(ql: Qiling): ql.hook_del(ep_hret) ql.hook_code(self.dbg_hook) @@ -36,6 +49,24 @@ def dbg_hook(self, ql: Qiling, address: int, size: int): if getattr(ql.arch, 'is_thumb', False): address |= 1 + # poll for an async interrupt from the client (gdb sends a bare \x03 while + # the target is running). throttled so the socket check stays off the hot + # path. this is the only way to break into a free-running guest, since the + # stub's packet loop is blocked inside emu_start until the target stops. + if self.check_interrupt is not None: + self._poll_counter += 1 + + if self._poll_counter >= self.INTR_POLL_INTERVAL: + self._poll_counter = 0 + + if self.check_interrupt(): + self.interrupted = True + self.last_bp = None + + ql.log.info(f'{PROMPT} interrupted by client, stopped at {address:#x}') + ql.stop() + return + # resuming emulation after hitting a breakpoint will re-enter this hook. # avoid an endless hooking loop by detecting and skipping this case if address == self.last_bp: @@ -83,4 +114,8 @@ def resume_emu(self, address: Optional[int] = None, steps: int = 0): op = f'stepping {steps} instructions' if steps else 'resuming' self.ql.log.info(f'{PROMPT} {op} from {address:#x}') + # clear any pending interrupt state from a previous resume + self.interrupted = False + self._poll_counter = 0 + self.ql.emu_start(address, self.exit_point, count=steps) diff --git a/tests/test_debugger.py b/tests/test_debugger.py index c8df8ae0c..17ae7b8d1 100644 --- a/tests/test_debugger.py +++ b/tests/test_debugger.py @@ -44,6 +44,61 @@ def send(self, msg: str): self.__file.flush() +class ReadingGdbClient: + """A minimal gdb remote client that can also read replies, so tests can + assert on the stop-reply packets the server sends back. + """ + + def __init__(self, host: str, port: int): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect((host, port)) + sock.settimeout(10) + + self.__sock = sock + self.__buf = b'' + + def __enter__(self): + return self + + def __exit__(self, ex_type, ex_value, ex_traceback): + self.__sock.close() + + @staticmethod + def checksum(data: str) -> int: + return sum(ord(c) for c in data) & 0xff + + def send(self, msg: str): + self.__sock.sendall(f'${msg}#{ReadingGdbClient.checksum(msg):02x}'.encode('latin')) + + def send_break(self): + """Send an async interrupt: a bare '\x03' byte, outside of packet framing. + """ + + self.__sock.sendall(b'\x03') + + def read_packet(self) -> str: + """Read a single '$#' reply, skipping any '+'/'-' acks. + """ + + # wait for a packet start marker + while b'$' not in self.__buf: + self.__buf += self.__sock.recv(4096) + + start = self.__buf.index(b'$') + + # wait until the terminating '#' and its two checksum digits arrived + while True: + end = self.__buf.find(b'#', start) + if end != -1 and len(self.__buf) >= end + 3: + break + self.__buf += self.__sock.recv(4096) + + data = self.__buf[start + 1:end] + self.__buf = self.__buf[end + 3:] + + return data.decode('latin') + + class DebuggerTest(unittest.TestCase): def test_gdbdebug_file_server(self): ql = Qiling(["../examples/rootfs/x8664_linux/bin/x8664_hello"], "../examples/rootfs/x8664_linux", verbose=QL_VERBOSE.DEBUG) @@ -168,6 +223,141 @@ def gdb_test_client(): ql.run() del ql + def test_gdbdebug_stepi_reports_sigtrap(self): + # regression for issues #1377 and #1538: a single-step ('s') used to be + # answered with a SIGTERM stop-reply because emu_state is always STOPPED + # after a step, which made gdb clients believe the program had died. the + # stop-reply for an ordinary step must be a SIGTRAP ('S05'), never a + # SIGTERM ('S0f'). + ql = Qiling(["../examples/rootfs/x8664_linux/bin/x8664_hello"], "../examples/rootfs/x8664_linux", verbose=QL_VERBOSE.OFF) + ql.debugger = 'gdb:127.0.0.1:9996' + + replies = [] + + def gdb_test_client(): + # yield to allow ql to launch its gdbserver + time.sleep(1.337 * 2) + + with ReadingGdbClient('127.0.0.1', 9996) as client: + client.send('qSupported:multiprocess+;swbreak+;hwbreak+;vContSupported+;xmlRegisters=i386') + client.read_packet() + client.send('QStartNoAckMode') + client.read_packet() + + # step a few instructions; every reply must be a SIGTRAP stop + for _ in range(3): + client.send('s') + replies.append(client.read_packet()) + + client.send('k') + + thread = threading.Thread(target=gdb_test_client, daemon=True) + thread.start() + + ql.run() + thread.join(timeout=30) + del ql + + self.assertEqual(replies, ['S05', 'S05', 'S05']) + + def test_gdbdebug_vcont_signal_actions(self): + # regression for issue #1377: a client resuming with a pending signal sends + # a 'vCont;S' or 'vCont;C' action (e.g. 'S0f' after a stop-reply we + # sent). the stub only recognized 'S05' and 'C05', replied empty to anything + # else, and the client bailed out with 'Invalid remote reply'. any signal must + # be accepted and carried out as a plain step or resume. + ql = Qiling(["../examples/rootfs/x8664_linux/bin/x8664_hello"], "../examples/rootfs/x8664_linux", verbose=QL_VERBOSE.OFF) + ql.debugger = 'gdb:127.0.0.1:9995' + + replies = [] + + def gdb_test_client(): + # yield to allow ql to launch its gdbserver + time.sleep(1.337 * 2) + + with ReadingGdbClient('127.0.0.1', 9995) as client: + client.send('qSupported:multiprocess+;swbreak+;hwbreak+;vContSupported+;xmlRegisters=i386') + client.read_packet() + client.send('QStartNoAckMode') + client.read_packet() + + client.send('vCont?') + replies.append(client.read_packet()) + + # step while delivering SIGTERM; this is the packet reported in #1377 + client.send('vCont;S0f:p1.1;c:p1.-1') + replies.append(client.read_packet()) + + # resume while delivering SIGTERM; runs the guest to completion + client.send('vCont;C0f:p1.1') + replies.append(client.read_packet()) + + client.send('k') + + thread = threading.Thread(target=gdb_test_client, daemon=True) + thread.start() + + ql.run() + thread.join(timeout=30) + del ql + + self.assertEqual(replies[0], 'vCont;c;C;s;S') + + # the signalled step is an ordinary step: SIGTRAP, not an empty reply + self.assertEqual(replies[1], 'S05') + + # the signalled resume ran to termination and reported an exit code + self.assertTrue(replies[2].startswith('W'), f'unexpected reply to signalled resume: {replies[2]!r}') + + def test_gdbdebug_async_interrupt(self): + # a free-running guest could not be interrupted at all: the stub is blocked + # inside emu_start while the target runs, so the bare '\x03' break byte a + # client sends to pause it was never read, and clients reported 'Cannot + # execute this command while the target is running'. a break must stop the + # guest and yield a SIGINT ('S02') stop-reply. + INFINITE_LOOP = bytes.fromhex('90ebfd') # nop ; jmp -3 + + ql = Qiling(code=INFINITE_LOOP, archtype=QL_ARCH.X8664, ostype=QL_OS.LINUX, verbose=QL_VERBOSE.OFF) + ql.debugger = 'gdb:127.0.0.1:9994' + + replies = [] + + def gdb_test_client(): + # yield to allow ql to launch its gdbserver + time.sleep(1.337 * 2) + + with ReadingGdbClient('127.0.0.1', 9994) as client: + client.send('qSupported:multiprocess+;swbreak+;hwbreak+;vContSupported+;xmlRegisters=i386') + client.read_packet() + client.send('QStartNoAckMode') + client.read_packet() + + # let the guest free-run; it never stops on its own + client.send('c') + + # give it time to spin, then break into it + time.sleep(1.337) + client.send_break() + + try: + replies.append(client.read_packet()) + except OSError: + # the stub ignored the break and the guest is still spinning. + # stop it from here so the assertion below reports the failure + # instead of hanging the test run forever + ql.stop() + + client.send('k') + + thread = threading.Thread(target=gdb_test_client, daemon=True) + thread.start() + + ql.run() + thread.join(timeout=30) + del ql + + self.assertEqual(replies, ['S02']) + def test_gdbdebug_shellcode_server(self): X8664_LIN = bytes.fromhex('31c048bbd19d9691d08c97ff48f7db53545f995257545eb03b0f05')