diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..458ea2f --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,83 @@ +name: Release + +on: + push: + tags: + - "v*" + workflow_dispatch: + +permissions: + contents: write + +jobs: + build: + name: Build ${{ matrix.name }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - name: linux-x86_64 + os: ubuntu-22.04 + - name: macos-arm64 + os: macos-14 + - name: macos-x86_64 + os: macos-13 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Rust + run: | + rustup update stable + rustup default stable + + - name: Install Linux dependencies + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y libfuse3-dev libfuse-dev pkg-config + + - name: Install macOS dependencies + if: runner.os == 'macOS' + run: | + brew install pkgconf + brew install --cask macfuse + + - name: Test + run: cargo test --locked + + - name: Package + run: ./scripts/package-release.sh + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.name }} + path: | + target/dist/*.tar.gz + target/dist/*.sha256 + + publish: + name: Publish GitHub Release + needs: build + if: startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-22.04 + steps: + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + path: dist + merge-multiple: true + + - name: Publish release + env: + GH_TOKEN: ${{ github.token }} + TAG_NAME: ${{ github.ref_name }} + run: | + if gh release view "$TAG_NAME" >/dev/null 2>&1; then + gh release upload "$TAG_NAME" dist/*.tar.gz dist/*.sha256 --clobber + else + gh release create "$TAG_NAME" dist/*.tar.gz dist/*.sha256 --generate-notes + fi diff --git a/README.md b/README.md index dda230e..07fb76f 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,10 @@ This is a user-space filesystem for exposing Cybershuttle data sources. +## Google Summer of Code 2026 Final Report + +See [GSoC-2026-Final-Report.md](./GSoC-2026-Final-Report.md). + This version loads ATLAS metadata from a TSV file and can also load mdCATH, MemProtMD, and GPCRmd metadata from TSV or simple CSV files. Each dataset entry is exposed as a directory containing a `metadata.json` file. @@ -39,6 +43,15 @@ also be materialized to a real directory when a static export is useful. dataset should be connected. The same official list is also exposed as the `datasets/` directory, where each dataset has a `metadata.json` file. +## Documentation + +- [User Guide](docs/user-guide.md): install, configure, mount, browse, and + troubleshoot the filesystem. +- [Developer Guide](docs/developer-guide.md): repo structure, architecture, + registry model, connector interface, and local development. +- [Distribution](docs/distribution.md): local packaging and GitHub Release + workflow for macOS and Linux binaries. + ## Official Dataset Registry The filesystem includes an official registry for the AI-for-science datasets diff --git a/docs/developer-guide.md b/docs/developer-guide.md new file mode 100644 index 0000000..c54c617 --- /dev/null +++ b/docs/developer-guide.md @@ -0,0 +1,189 @@ +# Developer Guide + +`cs-filesystem` is a read-only filesystem for exposing Cybershuttle dataset +metadata as ordinary directories and JSON files. + +## Repository Structure + +```text +Cargo.toml Rust package metadata and dependencies +Cargo.lock Locked dependency graph +README.md Project overview and quick start +docs/ Distribution, developer, and user documentation +examples/ Tiny metadata tables for demos and tests +scripts/package-release.sh Local release archive builder +src/main.rs CLI, FUSE filesystem, materialization path, NFS wiring +src/atlas.rs ATLAS TSV parser and virtual data source +src/table_dataset.rs Generic TSV/CSV-backed data source +src/dataset_registry.rs Built-in official dataset registry +src/nfs_server.rs Read-only NFS server implementation +``` + +The `data/` directory is intentionally local-only. Full metadata tables can be +large or subject to external access rules, so only small examples are committed. + +## Architecture + +The filesystem has one logical tree that can be exposed three ways: + +- `fuse`: mount the virtual tree through FUSE. +- `materialize`: write the same tree to a normal directory. +- `nfs`: serve the same metadata tree through a read-only NFS server. + +The root contains: + +```text +index.json +registry.json +datasets/ +atlas/ +/ +``` + +Each dataset root contains one directory per entry. Each entry directory contains +a `metadata.json` file. + +```text +atlas/ + 1r6w_A/ + metadata.json +``` + +## Core Interfaces + +`src/main.rs` defines the shared data source interface: + +```rust +pub trait VirtualDataSource { + fn name(&self) -> &str; + fn kind(&self) -> &str; + fn inode(&self) -> u64; + fn entry_count(&self) -> usize; + + fn lookup(&self, parent: u64, name: &OsStr) -> Option; + fn getattr(&self, ino: u64) -> Option; + fn read(&self, ino: u64, offset: i64, size: u32) -> Option>; + fn readdir(&self, ino: u64, offset: i64, reply: &mut ReplyDirectory) -> bool; +} +``` + +FUSE uses this trait directly. `materialize` uses each data source's +materialization helper. NFS converts data sources into `NfsDataset` values and +serves them through `src/nfs_server.rs`. + +Inodes are allocated by `InodeGenerator` in one pass while building data +sources. The root-level inode values in `main.rs` are reserved for +`/`, `hello.txt`, `index.json`, and `registry.json`; generated dataset inodes +start after those values. + +## Metadata Registry + +`src/dataset_registry.rs` contains the built-in registry of known AI-for-science +datasets. It exposes the registry in two forms: + +- `registry.json`: one JSON file containing all registry entries. +- `datasets//metadata.json`: one directory per registry entry. + +Registry entries include: + +- stable ID +- display name +- domain +- access pattern +- implementation status +- source URL +- intended connector type +- notes + +The registry is metadata-first. Planned or restricted datasets should be listed +without implying that raw data will be downloaded automatically. + +## Connector Model + +Current connectors are implemented as data sources: + +- `AtlasDataSource`: parses ATLAS-specific TSV metadata. +- `TableDataSource`: parses generic TSV or CSV metadata, using the first column + as the entry directory name. + +The registry `connector` field describes the expected access pattern for future +connectors, such as `api`, `bundle`, `zenodo_metadata`, or `table`. A connector +should expose metadata first and avoid surprising bulk downloads during normal +mount or demo flows. + +## Add A New Dataset From A Table + +If the dataset can be represented by a TSV or CSV table, no code change is +required. Put a stable entry ID in the first column: + +```text +entry_id family source +sample_001 protein local-demo +``` + +Run: + +```bash +cargo run --release -- materialize \ + examples/atlas_sample.tsv \ + my_dataset=/path/to/my_dataset.tsv \ + /tmp/cs_export +``` + +The output will include: + +```text +/tmp/cs_export/my_dataset/sample_001/metadata.json +``` + +## Add A New Data Source In Code + +Use a dedicated data source when a dataset needs custom parsing, remote API +metadata, caching, or non-table layout. + +1. Add a module under `src/`. +2. Parse external metadata into stable entry records. +3. Allocate inodes with `InodeGenerator`. +4. Implement `VirtualDataSource`. +5. Add a `materialize` helper if the source should support static export. +6. Wire the source into `main.rs`. +7. Add or update a registry entry in `dataset_registry.rs`. +8. Add unit tests for parsing, lookup, read slicing, and directory listing. + +Keep data sources read-only unless the project explicitly adds a write contract. + +## Build And Test Locally + +Install dependencies: + +Linux: + +```bash +sudo apt install cargo libfuse3-dev libfuse-dev pkg-config +``` + +macOS: + +```bash +brew install pkgconf +brew install --cask macfuse +``` + +Build and test: + +```bash +cargo test +cargo build --release +``` + +Run a sample export: + +```bash +rm -rf /tmp/cs_sample_export +cargo run --release -- materialize \ + examples/atlas_sample.tsv \ + mdcath=examples/mdcath_sample.tsv \ + memprotmd=examples/memprotmd_sample.tsv \ + gpcrmd=examples/gpcrmd_sample.tsv \ + /tmp/cs_sample_export +``` diff --git a/docs/distribution.md b/docs/distribution.md new file mode 100644 index 0000000..e7f47df --- /dev/null +++ b/docs/distribution.md @@ -0,0 +1,82 @@ +# Distribution + +This project distributes `cs-filesystem` as platform-specific `.tar.gz` +archives. Each archive contains the binary, examples, and documentation. + +## Supported Targets + +- Linux x86_64 +- macOS arm64 +- macOS x86_64 + +The binary depends on the host FUSE stack when running in `fuse` mode: + +- Linux: `libfuse3`/`libfuse` +- macOS: macFUSE + +`materialize` and `nfs` mode do not require users to mount with FUSE, but the +binary is built with FUSE support. + +## Local Package + +Build and package the current host platform: + +```bash +./scripts/package-release.sh +``` + +Outputs are written under: + +```text +target/dist/ +``` + +The archive name is: + +```text +cs-filesystem--.tar.gz +``` + +## GitHub Releases + +The release workflow lives at `.github/workflows/release.yml`. + +To publish a release after reviewing local changes: + +```bash +git tag v0.1.0 +git push origin v0.1.0 +``` + +The workflow will: + +1. Build and test on Linux and macOS runners. +2. Package each platform binary. +3. Upload archives and SHA-256 checksum files to a GitHub Release. + +The workflow can also be run manually with `workflow_dispatch`; manual runs build +artifacts but only tag pushes publish a GitHub Release. + +## Pre-release Checklist + +1. Update `version` in `Cargo.toml`. +2. Run `cargo test --locked`. +3. Run `cargo build --release`. +4. Run `./scripts/package-release.sh` locally. +5. Smoke test sample materialization: + +```bash +rm -rf /tmp/cs_sample_export +./target/release/cs-filesystem materialize \ + examples/atlas_sample.tsv \ + mdcath=examples/mdcath_sample.tsv \ + memprotmd=examples/memprotmd_sample.tsv \ + gpcrmd=examples/gpcrmd_sample.tsv \ + /tmp/cs_sample_export + +cat /tmp/cs_sample_export/index.json +cat /tmp/cs_sample_export/atlas/1r6w_A/metadata.json +``` + +6. Smoke test FUSE or NFS on at least one target host when possible. +7. Create and push a `vX.Y.Z` tag. diff --git a/docs/user-guide.md b/docs/user-guide.md new file mode 100644 index 0000000..3beeae3 --- /dev/null +++ b/docs/user-guide.md @@ -0,0 +1,181 @@ +# User Guide + +`cs-filesystem` exposes dataset metadata as a read-only filesystem. You can use +it in three modes: + +- `materialize`: write the tree to a normal directory. +- `fuse`: mount the tree as a live FUSE filesystem. +- `nfs`: serve the tree through a read-only NFS server. + +## Install + +Download the archive for your platform from GitHub Releases, then install the +binary: + +```bash +tar -xzf cs-filesystem--.tar.gz +cd cs-filesystem-- +chmod +x cs-filesystem +sudo install -m 0755 cs-filesystem /usr/local/bin/cs-filesystem +``` + +From source: + +```bash +cargo build --release +sudo install -m 0755 target/release/cs-filesystem /usr/local/bin/cs-filesystem +``` + +## System Requirements + +Linux: + +```bash +sudo apt install libfuse3-dev libfuse-dev pkg-config +``` + +macOS: + +```bash +brew install pkgconf +brew install --cask macfuse +``` + +macFUSE may require approval in System Settings before mounts work. + +## Metadata Tables + +ATLAS metadata is passed as the first table argument. Additional datasets can be +passed as `name=path` table specs. + +The examples bundled with the repo are enough for a quick demo: + +```text +examples/atlas_sample.tsv +examples/mdcath_sample.tsv +examples/memprotmd_sample.tsv +examples/gpcrmd_sample.tsv +``` + +For your own tables, use TSV or simple CSV. The first column becomes the entry +directory name, and every row becomes a `metadata.json` file. + +## Materialize A Directory + +This is the easiest mode for demos, inspection, and tools that only need a real +directory. + +```bash +rm -rf /tmp/cs_sample_export +cs-filesystem materialize \ + examples/atlas_sample.tsv \ + mdcath=examples/mdcath_sample.tsv \ + memprotmd=examples/memprotmd_sample.tsv \ + gpcrmd=examples/gpcrmd_sample.tsv \ + /tmp/cs_sample_export +``` + +Browse it: + +```bash +ls /tmp/cs_sample_export +cat /tmp/cs_sample_export/index.json +cat /tmp/cs_sample_export/registry.json +ls /tmp/cs_sample_export/atlas +cat /tmp/cs_sample_export/atlas/1r6w_A/metadata.json +``` + +## Mount With FUSE + +Create a mount directory: + +```bash +mkdir -p /tmp/cs_mount +``` + +Start the filesystem: + +```bash +cs-filesystem fuse \ + examples/atlas_sample.tsv \ + mdcath=examples/mdcath_sample.tsv \ + memprotmd=examples/memprotmd_sample.tsv \ + gpcrmd=examples/gpcrmd_sample.tsv \ + /tmp/cs_mount +``` + +Leave that command running. In another terminal: + +```bash +ls /tmp/cs_mount +cat /tmp/cs_mount/index.json +cat /tmp/cs_mount/datasets/alphafold_db/metadata.json +cat /tmp/cs_mount/memprotmd/1afo/metadata.json +``` + +Unmount on Linux: + +```bash +fusermount -u /tmp/cs_mount +``` + +Unmount on macOS: + +```bash +diskutil unmount /tmp/cs_mount +``` + +## Serve With NFS + +Start the read-only NFS server: + +```bash +cs-filesystem nfs \ + examples/atlas_sample.tsv \ + mdcath=examples/mdcath_sample.tsv \ + memprotmd=examples/memprotmd_sample.tsv \ + gpcrmd=examples/gpcrmd_sample.tsv \ + 127.0.0.1:2049 +``` + +Mounting the export depends on your host NFS client configuration and +permissions. Use this mode when another application or machine needs an NFS +view instead of a local FUSE mount. + +## Filesystem Layout + +```text +/ + index.json + registry.json + datasets/ + alphafold_db/ + metadata.json + atlas/ + 1r6w_A/ + metadata.json + mdcath/ + 1abcA00/ + metadata.json + memprotmd/ + 1afo/ + metadata.json + gpcrmd/ + adrb2_active/ + metadata.json +``` + +`index.json` lists the datasets loaded in this run. `registry.json` contains +the broader built-in dataset registry. `datasets/` exposes the registry as +ordinary directories. + +## Known Limitations + +- The filesystem is read-only. +- Metadata tables are parsed as simple TSV or CSV; quoted CSV fields are not yet + interpreted as structured CSV. +- The first column of each table must be a stable, filesystem-safe entry ID. +- Most registry datasets are metadata-only placeholders until dedicated + connectors are implemented. +- FUSE mode requires host FUSE support and may need elevated setup on macOS. +- Large raw datasets are intentionally not downloaded during normal mounts. diff --git a/scripts/package-release.sh b/scripts/package-release.sh new file mode 100755 index 0000000..b2e1f8a --- /dev/null +++ b/scripts/package-release.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -euo pipefail + +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$PROJECT_ROOT" + +BIN_NAME="cs-filesystem" +VERSION="${VERSION:-$(cargo metadata --no-deps --format-version 1 | sed -n 's/.*"version":"\([^"]*\)".*/\1/p' | head -1)}" +TARGET_TRIPLE="${TARGET_TRIPLE:-$(rustc -vV | sed -n 's/^host: //p')}" +ARCHIVE_BASENAME="${BIN_NAME}-${VERSION}-${TARGET_TRIPLE}" +STAGING_DIR="target/dist/${ARCHIVE_BASENAME}" + +cargo build --release + +rm -rf "$STAGING_DIR" +mkdir -p "$STAGING_DIR" + +cp "target/release/${BIN_NAME}" "$STAGING_DIR/" +cp README.md "$STAGING_DIR/" +cp -R docs "$STAGING_DIR/" +cp -R examples "$STAGING_DIR/" + +cat > "$STAGING_DIR/INSTALL.md" <<'INSTALL' +# Install + +Copy `cs-filesystem` somewhere on your PATH: + +```bash +chmod +x cs-filesystem +sudo install -m 0755 cs-filesystem /usr/local/bin/cs-filesystem +``` + +See `docs/user-guide.md` for mount, materialize, and NFS usage. +INSTALL + +mkdir -p target/dist +tar -C target/dist -czf "target/dist/${ARCHIVE_BASENAME}.tar.gz" "$ARCHIVE_BASENAME" + +if command -v shasum >/dev/null 2>&1; then + shasum -a 256 "target/dist/${ARCHIVE_BASENAME}.tar.gz" > "target/dist/${ARCHIVE_BASENAME}.tar.gz.sha256" +elif command -v sha256sum >/dev/null 2>&1; then + sha256sum "target/dist/${ARCHIVE_BASENAME}.tar.gz" > "target/dist/${ARCHIVE_BASENAME}.tar.gz.sha256" +fi + +echo "Wrote target/dist/${ARCHIVE_BASENAME}.tar.gz"