diff --git a/.dockerignore b/.dockerignore index 6dbac5d..ed5ee04 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,12 @@ .git .github .vscode +.env +.env.* +*.gguf +*.key +*.pem +*.safetensors llama-cpp-rs target *.md diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 86cdc66..0df5656 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,52 +1,69 @@ -name: Build and push Docker image +name: Build and publish Docker image on: push: branches: [main] - tags: ["v*"] workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: + contents: read + packages: write + jobs: build: runs-on: ubuntu-latest - permissions: - contents: read - packages: write steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + id: buildx + uses: docker/setup-buildx-action@v4 + + - name: Restore BuildKit cache mounts + id: cache-mounts + uses: actions/cache@v5 + with: + path: buildkit-cache-mounts + key: buildkit-cache-mounts-${{ runner.os }}-${{ github.sha }} + restore-keys: | + buildkit-cache-mounts-${{ runner.os }}- + + - name: Inject BuildKit cache mounts + uses: reproducible-containers/buildkit-cache-dance@v3 + with: + builder: ${{ steps.buildx.outputs.name }} + cache-dir: buildkit-cache-mounts + dockerfile: Dockerfile + skip-extraction: ${{ steps.cache-mounts.outputs.cache-hit }} - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract metadata + - name: Generate image metadata id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@v6 with: - images: ghcr.io/${{ github.repository_owner }}/ltengine + images: ghcr.io/${{ github.repository }} tags: | - type=ref,event=branch - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=sha,prefix=,suffix=,format=short + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} + type=sha,format=long,prefix= - name: Build and push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: context: . + file: Dockerfile push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=gha,scope=ltengine + cache-to: type=gha,mode=max,scope=ltengine diff --git a/Cargo.lock b/Cargo.lock index 0ebcf1f..1f03f32 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 4 [[package]] name = "actix-codec" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a" +checksum = "31404e1443b7b7bcaa311c1af456775cc3f668e34573f1503d7ce4844327629e" dependencies = [ "bitflags", "bytes", @@ -21,9 +21,9 @@ dependencies = [ [[package]] name = "actix-http" -version = "3.13.1" +version = "3.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48e2faa3e7418ed780cca54829d32782a4008a077230f67457caa063415e99c2" +checksum = "11004b0e9b44b4eb3d15e0c3132b96fb178c7e50a74758b2f17bb9cc9a7fb4f6" dependencies = [ "actix-codec", "actix-rt", @@ -85,7 +85,7 @@ dependencies = [ "log", "memchr", "mime", - "rand 0.8.7", + "rand 0.8.8", "serde", "serde_json", "serde_plain", @@ -123,9 +123,9 @@ dependencies = [ [[package]] name = "actix-rt" -version = "2.11.0" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92589714878ca59a7626ea19734f0e07a6a875197eec751bb5d3f99e64998c63" +checksum = "6a16bf2f19c2ad84842bdfe6f3665620e93197d5c607889bfa3aaac45e762fd1" dependencies = [ "futures-core", "tokio", @@ -133,9 +133,9 @@ dependencies = [ [[package]] name = "actix-server" -version = "2.7.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3716aae056e2f869b7b5cfd8a08fcf98890f8455bec61d69c7dff5d8576f9d2b" +checksum = "5d44ae8a6516f4ac7bfc7b61aabcd286104e96b4b24c747ce220832a016056d9" dependencies = [ "actix-rt", "actix-service", @@ -170,9 +170,9 @@ dependencies = [ [[package]] name = "actix-web" -version = "4.14.0" +version = "4.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df09e2d9239703dd64056359c920c7f3fba6535ec61a0059e0f44e095ffe02b4" +checksum = "bbacab3593b6b4f7be815076fc52d60a83c873426824675417e2abdd229e2e36" dependencies = [ "actix-codec", "actix-http", @@ -425,9 +425,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.1" +version = "1.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9066c49992464636f92905fa096ec58baaa4d57ec19a5c096c68d3e25ef3d136" +checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" dependencies = [ "find-msvc-tools", "jobserver", @@ -452,9 +452,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "chacha20" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", "cpufeatures", @@ -513,7 +513,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -600,18 +600,18 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" dependencies = [ "libc", ] [[package]] name = "crc32fast" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" dependencies = [ "cfg-if", ] @@ -742,14 +742,14 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] name = "either" -version = "1.17.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" [[package]] name = "encode_unicode" @@ -810,9 +810,9 @@ checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "find-msvc-tools" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" [[package]] name = "find_cuda_helper" @@ -825,12 +825,13 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" dependencies = [ "crc32fast", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -871,27 +872,27 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-sink" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] name = "futures-task" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-util" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-core", "futures-task", @@ -979,7 +980,7 @@ dependencies = [ "indicatif", "log", "native-tls", - "rand 0.8.7", + "rand 0.8.8", "serde", "serde_json", "thiserror 1.0.69", @@ -1020,9 +1021,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" dependencies = [ "displaydoc", "potential_utf", @@ -1034,9 +1035,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", @@ -1047,9 +1048,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -1061,16 +1062,17 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" [[package]] name = "icu_properties" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" dependencies = [ + "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -1081,15 +1083,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" [[package]] name = "icu_provider" -version = "2.2.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" dependencies = [ "displaydoc", "icu_locale_core", @@ -1135,9 +1137,9 @@ checksum = "277ff51754a3f68f12f58446c5d006aa8baa4914ea273cce24a599cfaff33d4f" [[package]] name = "indexmap" -version = "2.14.0" +version = "2.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +checksum = "07aa2048142242915a31d35844fb311e0e53fcca590c3a0a40dcf1b841fa09eb" dependencies = [ "equivalent", "hashbrown 0.17.1", @@ -1189,9 +1191,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.103" +version = "0.3.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" dependencies = [ "cfg-if", "futures-util", @@ -1222,9 +1224,9 @@ dependencies = [ [[package]] name = "libredox" -version = "0.1.19" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2026a5056764a10b2bf5d56488cba40da507f5493a6a429340e2004d9ed085fa" +checksum = "d7955dfc218a8afb29dfeffd540e3a6e96baeb94fe7138228dd7cc6937fbbf96" dependencies = [ "libc", ] @@ -1237,29 +1239,29 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" [[package]] name = "llama-cpp-2" -version = "0.1.154" +version = "0.1.155" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4397360a0bd81be29929a55ad2f49eba27a744a6578f3271894d6f41d86bd15b" +checksum = "a487906c1bab58ccc714a99934279a5626f2ca7ba76cbefa39d98dade3bf88a7" dependencies = [ "encoding_rs", "enumflags2", "llama-cpp-sys-2", - "thiserror 2.0.19", + "thiserror 2.0.20", "tracing", "tracing-core", ] [[package]] name = "llama-cpp-sys-2" -version = "0.1.154" +version = "0.1.155" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13a9ea2ce0cdc20bcb1870534022e340b391663f8fe09133951e2fe37fbc29cf" +checksum = "d59ed2a770245cf16526a8643c79872701fc6115556e5d8e675d9248752a1394" dependencies = [ "bindgen", "cc", @@ -1297,9 +1299,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "ltengine" @@ -1352,9 +1354,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.9" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" dependencies = [ "adler2", "simd-adler32", @@ -1536,21 +1538,21 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pkg-config" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "portable-atomic" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "potential_utf" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ "zerovec", ] @@ -1606,9 +1608,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" dependencies = [ "libc", "rand_chacha", @@ -1774,9 +1776,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "ring", "rustls-pki-types", @@ -1875,7 +1877,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -2026,9 +2028,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" dependencies = [ "proc-macro2", "quote", @@ -2070,11 +2072,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.19" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "thiserror-impl 2.0.19", + "thiserror-impl 2.0.20", ] [[package]] @@ -2090,13 +2092,13 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.19" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -2131,9 +2133,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", "zerovec", @@ -2323,9 +2325,9 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasm-bindgen" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" dependencies = [ "cfg-if", "once_cell", @@ -2336,9 +2338,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2346,9 +2348,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" dependencies = [ "bumpalo", "proc-macro2", @@ -2359,9 +2361,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" dependencies = [ "unicode-ident", ] @@ -2578,9 +2580,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "writeable" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "yoke" @@ -2654,9 +2656,9 @@ checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" [[package]] name = "zerotrie" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" dependencies = [ "displaydoc", "yoke", @@ -2665,9 +2667,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" dependencies = [ "yoke", "zerofrom", @@ -2676,15 +2678,21 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.3" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.4", ] +[[package]] +name = "zlib-rs" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" + [[package]] name = "zmij" version = "1.0.23" diff --git a/Dockerfile b/Dockerfile index 8521de3..cf08ee7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,46 +1,80 @@ # syntax=docker/dockerfile:1 -# Build (CUDA dev toolkit + Rust + clang/cmake for llama.cpp) -FROM nvidia/cuda:13.3.0-devel-ubuntu24.04 AS builder +ARG CUDA_VERSION=13.3.1 +ARG UBUNTU_VERSION=26.04 -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y --no-install-recommends \ - curl build-essential clang cmake pkg-config libssl-dev \ - && rm -rf /var/lib/apt/lists/* \ - && rm -f /usr/local/cuda/include/nccl*.h +FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION} AS builder -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable -ENV PATH="/root/.cargo/bin:${PATH}" +ARG DEBIAN_FRONTEND=noninteractive +ARG RUST_VERSION=1.98.0 + +ENV RUSTUP_HOME=/root/.rustup \ + CARGO_HOME=/root/.cargo \ + PATH=/root/.cargo/bin:${PATH} + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + build-essential \ + clang \ + cmake \ + pkg-config \ + libssl-dev \ + && rm -rf /var/lib/apt/lists/* + +RUN curl --proto '=https' --tlsv1.2 -sSf \ + https://sh.rustup.rs \ + -o /tmp/rustup-init.sh \ + && sh /tmp/rustup-init.sh \ + -y \ + --profile minimal \ + --default-toolchain "${RUST_VERSION}" \ + && rm /tmp/rustup-init.sh WORKDIR /build -# Copy LTEngine source COPY . . -# Build with CUDA support (cargo caches survive layer cache hits) -RUN --mount=type=cache,id=ltengine-cargo-git,target=/usr/local/cargo/git \ - --mount=type=cache,id=ltengine-cargo-registry,target=/root/.cargo/registry \ - --mount=type=cache,id=ltengine-target-cuda13-nonccl,target=/build/target \ - cargo build --features cuda --release -p ltengine && \ - cp target/release/ltengine /ltengine +RUN --mount=type=cache,id=ltengine-cargo-registry,target=/root/.cargo/registry,sharing=locked \ + --mount=type=cache,id=ltengine-cargo-git,target=/root/.cargo/git,sharing=locked \ + --mount=type=cache,id=ltengine-target-cuda1331-ubuntu2604-nonccl,target=/build/target \ + cargo build \ + --locked \ + --features cuda \ + --release \ + -p ltengine \ + && install -Dm755 -s target/release/ltengine /out/ltengine + -# Runtime (lean: only CUDA runtime libs, no build tools) -FROM nvidia/cuda:13.3.0-runtime-ubuntu24.04 +FROM nvidia/cuda:${CUDA_VERSION}-base-ubuntu${UBUNTU_VERSION} -RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates libssl3 libgomp1 \ +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + libssl3t64 \ + libgomp1 \ && rm -rf /var/lib/apt/lists/* -RUN useradd --system --no-create-home ltengine \ +RUN useradd \ + --system \ + --no-create-home \ + --shell /usr/sbin/nologin \ + ltengine \ && mkdir -p /models \ && chown ltengine:ltengine /models -COPY --from=builder /ltengine /usr/local/bin/ltengine +COPY --from=builder /out/ltengine /usr/local/bin/ltengine + +ENV HF_HOME=/models \ + LTENGINE_MODEL=gemma3-4b -ENV HF_HOME=/models -ENV LTENGINE_MODEL=gemma3-4b VOLUME ["/models"] + EXPOSE 5050 USER ltengine -CMD ["sh", "-c", "exec ltengine --host 0.0.0.0 -m \"${LTENGINE_MODEL}\" ${LTENGINE_MODEL_FILE:+--model-file \"${LTENGINE_MODEL_FILE}\"}"] + +CMD ["sh", "-c", "set -eu; set -- ltengine --host 0.0.0.0 -m \"$LTENGINE_MODEL\"; if [ -n \"${LTENGINE_MODEL_FILE:-}\" ]; then set -- \"$@\" --model-file \"$LTENGINE_MODEL_FILE\"; fi; exec \"$@\""] diff --git a/ltengine/Cargo.toml b/ltengine/Cargo.toml index 9883507..fd49b7a 100644 --- a/ltengine/Cargo.toml +++ b/ltengine/Cargo.toml @@ -15,7 +15,7 @@ hf-hub = { version = "0.3.2" } anyhow = "1.0.104" thiserror = { workspace = true } parking_lot = "0.12.5" -llama-cpp-2 = { version = "0.1.154" } +llama-cpp-2 = { version = "0.1.155" } encoding_rs = "0.8.35" actix-multipart = "0.7.2" whatlang = "0.16.4" diff --git a/ltengine/src/llm.rs b/ltengine/src/llm.rs index 381aa2a..4f55090 100644 --- a/ltengine/src/llm.rs +++ b/ltengine/src/llm.rs @@ -244,7 +244,7 @@ impl LLMContext<'_>{ let seq_breakers = vec![b"\n", b":", b"\"", b"*"]; let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(64, 1.0, 0.0, 0.0), + LlamaSampler::penalties(self.llm.model.n_vocab(), 64, 1.0, 0.0, 0.0), LlamaSampler::dry(&self.llm.model, 0.0, 1.75, 2, -1, seq_breakers), LlamaSampler::top_k(40), LlamaSampler::typical(1.0, 0),