diff --git a/.editorconfig b/.editorconfig index 8df04e3..e464fc3 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,5 +1,5 @@ # SPDX-License-Identifier: MIT AND LicenseRef-Palimpsest-0.8 -# SPDX-FileCopyrightText: 2024-2025 Ehsaneddin Asgari and Contributors +# SPDX-FileCopyrightText: 2024-2026 Jonathan D.A. Jewell and Contributors # EditorConfig for 1000Langs # https://editorconfig.org diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 5a34322..3927684 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing to 1000Langs @@ -170,7 +170,7 @@ Then create a pull request on GitHub. ``` rescript // SPDX-License-Identifier: MPL-2.0 -// SPDX-FileCopyrightText: 2024-2025 Ehsaneddin Asgari and Contributors +// SPDX-FileCopyrightText: 2024-2026 Jonathan D.A. Jewell and Contributors /** * Module documentation goes here. diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bd26e99..f1da774 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,5 @@ # SPDX-License-Identifier: MPL-2.0 -# SPDX-FileCopyrightText: 2024-2025 Ehsaneddin Asgari and Contributors +# SPDX-FileCopyrightText: 2024-2026 Jonathan D.A. Jewell and Contributors # # 1000Langs GitLab CI/CD Configuration # ===================================== diff --git a/CODE_OF_CONDUCT.adoc b/CODE_OF_CONDUCT.adoc index 7081e8a..bfa5440 100644 --- a/CODE_OF_CONDUCT.adoc +++ b/CODE_OF_CONDUCT.adoc @@ -1,5 +1,5 @@ // SPDX-License-Identifier: CC-BY-SA-4.0 -// SPDX-FileCopyrightText: 2024-2025 Ehsaneddin Asgari and Contributors +// SPDX-FileCopyrightText: 2024-2026 Jonathan D.A. Jewell and Contributors = Code of Conduct :toc: :toclevels: 2 diff --git a/Containerfile b/Containerfile index 982d739..4b62695 100644 --- a/Containerfile +++ b/Containerfile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: MIT AND LicenseRef-Palimpsest-0.8 -# SPDX-FileCopyrightText: 2024-2025 Ehsaneddin Asgari and Contributors +# SPDX-FileCopyrightText: 2024-2026 Jonathan D.A. Jewell and Contributors # # 1000Langs Containerfile (Podman-native) # ======================================= diff --git a/FUTURE.adoc b/FUTURE.adoc index 0e76060..3eb0861 100644 --- a/FUTURE.adoc +++ b/FUTURE.adoc @@ -1,5 +1,5 @@ // SPDX-License-Identifier: CC-BY-SA-4.0 -// SPDX-FileCopyrightText: 2024-2025 Ehsaneddin Asgari and Contributors +// SPDX-FileCopyrightText: 2024-2026 Jonathan D.A. Jewell and Contributors = Future Roadmap :toc: :toclevels: 2 diff --git a/LICENSES/Apache-2.0.txt b/LICENSES/Apache-2.0.txt new file mode 100755 index 0000000..24e8eee --- /dev/null +++ b/LICENSES/Apache-2.0.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [2018] [Ehsaneddin Asgari] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..5fc8b11 --- /dev/null +++ b/NOTICE @@ -0,0 +1,41 @@ +LOL +Copyright 2024-2026 Jonathan D.A. Jewell and Contributors + +This project contains data files derived from 1000Langs +(https://github.com/ehsanasgari/1000Langs), Copyright (c) 2018-2019 +Ehsaneddin Asgari, licensed under the Apache License, Version 2.0. A copy of +that licence is in LICENSES/Apache-2.0.txt. 1000Langs has no NOTICE file of +its own, so there is nothing further to carry over. + +What was taken +-------------- +Every file under meta/ (8 data files and 196 flag images) is an unmodified +copy of the same path in 1000Langs. They are still under Apache-2.0. Some of +them are material that 1000Langs itself redistributed: captured web pages +(meta/bibles_list.html, meta/ebible.html), an API response +(meta/api_volumes.txt), a language table (meta/ISO639-3.tsv) and flag images +(meta/flags/). Their original authors are not recorded in 1000Langs and are +not asserted here. REUSE.toml lists the licensing file by file. + +What was not taken +------------------ +LOL's code contains no code from 1000Langs. It is an independent +reimplementation of the same idea, written first in ReScript and now in +AffineScript, rather than translated from the Python. The significant +differences are: + + * Language and runtime: AffineScript/ReScript on a JS runtime, plus an + Idris2 ABI, Zig FFI, Agda proofs, a Julia analysis layer and Nickel + configuration, in place of Python with requests, lxml and pandas. + * APIs: API.Bible v1 in place of the Digital Bible Platform v2 (dbt.io). + * Crawling: endpoint-based fetches per chapter with a rate limiter and + retry policy, in place of following "next" links with XPath. + * Data model: typed verse and chapter records in place of BBBCCCVVV-keyed + TSV files. + * Sources: eBible.org and Find.Bible are crawled for text; 1000Langs used + them only for metadata. + * Added: a VeriSimDB corpus-quality scan and an OpenCyc integration stub. + * Removed: the Massive Parallel corpus comparison, per-source crawl reports, + repeat-for-coverage loops and metadata refresh. + +The evidence for this split is in docs/provenance/1000langs-comparison.adoc. diff --git a/README.adoc b/README.adoc index c3f4856..3d57b47 100644 --- a/README.adoc +++ b/README.adoc @@ -1,5 +1,5 @@ // SPDX-License-Identifier: CC-BY-SA-4.0 -// SPDX-FileCopyrightText: 2024-2025 Ehsaneddin Asgari and Contributors +// SPDX-FileCopyrightText: 2024-2026 Jonathan D.A. Jewell and Contributors = LOL image:https://img.shields.io/badge/License-PMPL--1.0-blue.svg[License: PMPL-1.0,link="https://github.com/hyperpolymath/palimpsest-license"] @@ -129,6 +129,22 @@ just crawl-all |1200+ |=== +== Corpus Data and Translation Licences + +LOL crawls translations; it does not redistribute them. The texts it fetches +belong to their publishers, not to this project. + +* Many translations on Bible.com, API.Bible and similar sites are copyrighted + and are covered by those sites' terms of service. +* Do not republish a corpus built with LOL without checking the licence of + each translation in it. +* https://ebible.org[eBible.org] hosts many public-domain and openly licensed + translations and is the safest default source. + +Each source in `config/main.ncl` carries a `terms_url` field pointing at that +source's licence or terms page. A value of `"unknown"` means the page has not +been verified. + == Configuration Configuration is managed through Nickel for type-safe, validated settings: @@ -231,7 +247,12 @@ Commercial use with attribution is permitted. Proprietary AI training without at == Acknowledgments -* Original Python implementation by Ehsaneddin Asgari (LMU Munich) +* LOL is an independent reimplementation inspired by + https://github.com/ehsanasgari/1000Langs[1000Langs] by Ehsaneddin Asgari + (LMU Munich), licensed under the Apache License 2.0. LOL contains no code + from 1000Langs. The data files under `meta/` are unmodified copies from + 1000Langs and remain under Apache-2.0; see link:NOTICE[NOTICE] and + link:docs/provenance/1000langs-comparison.adoc[the provenance comparison]. * Bible corpus data from various open Bible translation projects * https://github.com/hyperpolymath/echidna[Echidna] for proof verification * https://github.com/Hyperpolymath/rhodium-standard-repositories[RSR] for compliance framework diff --git a/REUSE.toml b/REUSE.toml new file mode 100644 index 0000000..8db895f --- /dev/null +++ b/REUSE.toml @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell and Contributors +# +# Licensing for files that cannot carry an inline SPDX header. +# Every file under meta/ is an unmodified copy of the same path in +# https://github.com/ehsanasgari/1000Langs (identical git blob SHAs), which is +# licensed Apache-2.0. See NOTICE and docs/provenance/1000langs-comparison.adoc. + +version = 1 + +# Data compiled within 1000Langs by its author. +[[annotations]] +path = [ + "meta/biblecom.tsv", + "meta/books2numbers.txt", + "meta/massive_par_stat.tsv", + "meta/pngscript_filenames.txt", +] +precedence = "override" +SPDX-FileCopyrightText = "2018-2019 Ehsaneddin Asgari" +SPDX-License-Identifier = "Apache-2.0" + +# Captures of third-party material that 1000Langs redistributed: +# api_volumes.txt is a Digital Bible Platform API response, bibles_list.html is +# a saved Find-A-Bible page, ebible.html is a saved eBible.org page. They are +# listed under the licence LOL received them under. Their original copyright +# holders are not recorded here because they have not been established. +[[annotations]] +path = [ + "meta/api_volumes.txt", + "meta/bibles_list.html", + "meta/ebible.html", +] +precedence = "override" +SPDX-License-Identifier = "Apache-2.0" + +# Redistributed by 1000Langs without a stated source. Original authorship is +# not established, so no copyright holder is asserted. +[[annotations]] +path = [ + "meta/ISO639-3.tsv", + "meta/flags/**", +] +precedence = "override" +SPDX-License-Identifier = "Apache-2.0" diff --git a/config/main.ncl b/config/main.ncl index 3a10817..a5d1858 100644 --- a/config/main.ncl +++ b/config/main.ncl @@ -1,5 +1,5 @@ # SPDX-License-Identifier: MIT AND LicenseRef-Palimpsest-0.8 -# SPDX-FileCopyrightText: 2024-2025 Ehsaneddin Asgari and Contributors +# SPDX-FileCopyrightText: 2024-2026 Jonathan D.A. Jewell and Contributors # # 1000Langs Nickel Configuration # ============================== @@ -82,6 +82,7 @@ let default_source = { }, # Source configurations + # terms_url: the source's licence or terms page, or "unknown" where not verified. sources = { bible_cloud = default_source & { name = "Bible Cloud", @@ -90,12 +91,14 @@ let default_source = { requires_auth = true, auth_type = "api_key", auth_env_var = "BIBLE_CLOUD_API_KEY", + terms_url = "https://api.bible/terms-and-conditions", priority = 1, }, bible_com = default_source & { name = "Bible.com (YouVersion)", type = "scraper", base_url | Contracts.Url = "https://www.bible.com", + terms_url = "unknown", requires_auth = false, crawler_config = default_crawler_config & { rate_limit_ms = 2000, @@ -107,6 +110,7 @@ let default_source = { name = "PNG Scriptures", type = "download", base_url | Contracts.Url = "https://pngscriptures.org", + terms_url = "unknown", requires_auth = false, formats = ["zip", "html"], priority = 3, @@ -115,6 +119,7 @@ let default_source = { name = "eBible.org", type = "download", base_url | Contracts.Url = "https://ebible.org", + terms_url = "https://ebible.org/Scriptures/copyright.php", requires_auth = false, formats = ["usfm", "usx", "html"], priority = 4, @@ -123,6 +128,7 @@ let default_source = { name = "Find.Bible", type = "api", base_url | Contracts.Url = "https://find.bible", + terms_url = "unknown", requires_auth = false, priority = 5, }, diff --git a/docs/provenance/1000langs-comparison.adoc b/docs/provenance/1000langs-comparison.adoc new file mode 100644 index 0000000..cc8461c --- /dev/null +++ b/docs/provenance/1000langs-comparison.adoc @@ -0,0 +1,240 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 +// SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell and Contributors += LOL and 1000Langs: provenance comparison +:toc: preamble + +This document records what LOL took from +https://github.com/ehsanasgari/1000Langs[1000Langs] (Ehsaneddin Asgari, +Apache-2.0) and what it did not. It is the evidence behind `NOTICE`, +`REUSE.toml` and the Acknowledgments in `README.adoc`. + +== Conclusion + +*Mixed: the `meta/` data is derived (204 files copied verbatim); all code is +independent.* + +The crawler code is a reimplementation of the same idea. It targets the same +kind of sources but contains no code translated from the Python. Every file +under `meta/` is a byte-identical copy of the same path in 1000Langs and stays +under Apache-2.0. + +== What was compared + +[cols="1,3"] +|=== +|LOL +|`hyperpolymath/lol` `main` at `0cb776c`: `src/`, `lib/`, `meta/`, `config/`, +plus the ReScript originals at the root commit `b39c92a` + +|1000Langs +|`ehsanasgari/1000Langs` branch `master`: all 64 commits, 2018-08-28 to +2019-05-27. Every commit is by Asgari. `LICENSE` is Apache-2.0, +with Asgari named as holder for 2018 in its appendix. There is no `NOTICE` file. +|=== + +`lib/` holds only `lib/rescript.lock`, a build lock with no upstream +counterpart. + +== Data: `meta/` is a verbatim copy + +The pairs (git blob SHA, path) for every file under `meta/` were listed on both +sides with `git ls-tree -r -- meta` and compared with `diff`. The lists +are identical: 204 files, made up of 196 flag images under `meta/flags/` and +the 8 files below. Identical blob SHAs mean identical bytes. + +[cols="2,3,2"] +|=== +|File |What it is |How 1000Langs used it + +|`api_volumes.txt` +|A Digital Bible Platform (dbt.io) volume-list API response +|Read and rewritten by `bdpAPI/bdpAPI.py` (lines 56 and 132) + +|`biblecom.tsv` +|A table of Bible.com translation URLs, ISO codes and names +|Read and written by `bibleCOMAPI/biblecomAPI.py` (lines 40 and 86) + +|`bibles_list.html` +|A saved Find-A-Bible page ("Bibles \| Find-A-Bible") +|Parsed by `metaAPI/metadata.py` (line 27) + +|`books2numbers.txt` +|Book numbers with name and abbreviation variants +|Book-name lookup + +|`ebible.html` +|A saved eBible.org page ("Bible copyright information") +|Parsed by `metaAPI/metadata.py` (line 52) + +|`massive_par_stat.tsv` +|Per-language counts from the Massive Parallel corpus +|Written and read by `metaAPI/metadata.py` (lines 100-101) + +|`pngscript_filenames.txt` +|PNG Scriptures chapter file names (`MAT01.htm` …) +|Read by `biblecrawler/general_crawler.py` to drive the PNG crawl + +|`ISO639-3.tsv` +|A language table with columns `ISO NAM_LABEL LMP_LON LMP_LAT GENEALOGY` +|Language metadata +|=== + +Caveats on origin: + +* `bibles_list.html`, `ebible.html` and `api_volumes.txt` are captures of + third-party pages and API output that 1000Langs redistributed. +* `ISO639-3.tsv` is not SIL's plain code table: the coordinate and genealogy + columns suggest an Ethnologue-derived table. 1000Langs does not say where it + came from. +* The flags arrived in 1000Langs in commits "add new changes" (2018-12-15), + "Refactoring and improving the codes" (2018-12-18) and "remove unwanted + codes" (2018-12-20). None of them names a source. + +LOL received all of these under Apache-2.0, and that is the licence recorded in +`REUSE.toml`. No original copyright holder is asserted for the files above +whose origin is unestablished. + +No LOL source file reads `meta/`; only the `Containerfile` copies it into the +image. + +== Code: independent + +=== History + +* LOL's root commit `b39c92a` (2026-02-22, Jonathan D.A. Jewell, "Auto-commit: + Sync changes") adds the ReScript sources `src/*.res` in bulk. +* Commits titled "port to AffineScript" followed: `f3c2dea` … `f1e7263` + (2026-05-15) and `0cfe49d` (#611, 2026-08-24). These produced the current + `src/**/*.affine`. +* No Python file appears anywhere in LOL's history. Asgari is not an author of + any LOL commit, and the GitHub repository is not a fork. + +=== Upstream fingerprints, searched in the ReScript originals + +The comparison was made against the ReScript at `b39c92a` as well as the +machine-ported AffineScript, so a port could not hide a translation. In +`git show b39c92a:` across every `src/*.res`, the following upstream +identifiers and quirks have *zero* hits: + +`1188` (the PNG page count in `general_crawler.py:56`), `SLEEPTIME` +(`general_crawler.py:30`), `MAT.1` (`bibleCOM.py:49`), `BDPAPl` (the misspelt +class at `bdpAPI.py:23`), `dbt.io` (`bdpAPI.py:38`), `zfill`, `iterwalk` +(`biblePngscripturesORG.py:137`), `robot`, `Cysouw`, `dam_id`, `fcbh`, +`Asgari`, `python`. + +There are 3 hits for `xpath`, all for a generic `XPath(string)` selector +variant in `Crawler.res` and its test, not for upstream XPath expressions. The +name "1000Langs" appears 10 times, all as the project's name. + +=== Per-file comparison + +[cols="2,2,1,4"] +|=== +|LOL file |1000Langs counterpart |Verdict |Evidence on both sides + +|`src/Lang1000.affine` +|`run_crawler/lang1000.py` +|Inspired +|Both are CLI entry points. LOL's is AffineScript; upstream's is an +`argparse` script that drives per-site crawls and writes reports. + +|`src/crawlers/Crawler.affine` +|`biblecrawler/general_crawler.py` +|Inspired (weakly) +|LOL: typed request/response structs, a `RateLimiter` (line 91), a +`RetryPolicy` with backoff (line 113) and its own user agent (line 56). +Upstream: a `requests` session with a Firefox user agent (line 46) that +follows "next" links until a page repeats or 1188 pages are seen (line 56), +noted as "inspired/adapted from Michael Cysouw's crawling code" (line 13). + +|`src/crawlers/BibleCloud.affine` +|`biblecrawler/bibleCLOUD.py`, `bibleCLOUDAPI/biblecloudAPI.py` +|Inspired (name only) +|LOL calls the API.Bible v1 JSON API (`api.scripture.api.bible/v1`, lines +19-37). Upstream scrapes bible.cloud HTML with XPath +`//span[contains(@class,"v ")]` (lines 159, 163). + +|`src/crawlers/BibleCom.affine` +|`biblecrawler/bibleCOM.py`, `bibleCOMAPI/biblecomAPI.py` +|Inspired +|LOL matches CSS-module class `.ChapterContent_verse__uvbXo` (line 44) and a +`data-usfm` regex (line 136). Upstream uses XPath on `span.verse` (line 187) +and starts each crawl at `MAT.1` (lines 49, 64). + +|`src/crawlers/PngScriptures.affine` +|`biblecrawler/biblePngscripturesORG.py`, `biblePNGAPI/pngAPI.py` +|Inspired +|LOL fetches `/languages` and `/lng/` endpoints (lines 40-41) and parses +USFM `\v` lines or `data-verse` attributes (lines 131, 169). Upstream +downloads `_html.zip` archives (`pngAPI.py:66`) and walks them with +`etree.iterwalk` (line 137). + +|`src/crawlers/EBible.affine` +|`metaAPI/metadata.py` (`getMetaEbible`) +|Inspired source, independent code +|LOL downloads and parses USFM text (line 101). Upstream reads only metadata +from the saved `meta/ebible.html` (line 52) and never fetched eBible text. + +|`src/crawlers/FindBible.affine` +|`metaAPI/metadata.py` (`getMetaFindBible`) +|Inspired source, independent code +|LOL treats find.bible as a text source (line 19). Upstream parsed the saved +`meta/bibles_list.html` for metadata only (line 27). + +|`src/api/DigitalBiblePlatform.affine` +|`bdpAPI/bdpAPI.py` +|Inspired +|LOL uses API.Bible v1 with typed `verse` records (line 71). Upstream uses +dbt.io v2 (`apiversion`, `library/volume`, `library/verse`, lines 38, 130, +169) with `dam_id` volumes and BBBCCCVVV verse ids. + +|`src/utils/Http.affine`, `src/utils/Iso639.affine`, +`src/utils/Statistics.affine`, `src/verisimdb/*`, `src/cyc/OpenCyc.affine`, +`src/abi/*.idr`, `src/Lol/ABI` +|none +|Independent +|No counterpart in 1000Langs. +|=== + +"Inspired" means the LOL file targets the same site or role as the upstream +file but shares no logic, identifiers, selectors, endpoints or data layout with +it. Much of LOL's crawler code is scaffolding rather than a tested crawler. + +=== Prose + +`README.adoc` was compared with upstream `README.md` line by line after +normalising whitespace and case (`comm -12`). No line is shared. +`docs/CITATIONS.adoc` cites LOL and its author only. + +== The CHANGELOG claim + +`CHANGELOG.adoc` says LOL is a "complete rewrite from Python to ReScript", +that it "switched from Apache 2.0 to MIT + Palimpsest", and that a 0.0.1 +release was made in 2018 as Python under Apache-2.0. It links +`github.com/Hyperpolymath/1000Langs`, which returns 404. + +The repository's history does not support this: + +* there was never Python in LOL; +* the 2018 release is 1000Langs's history, not LOL's; +* the code shares nothing with the Python beyond its purpose. + +Read accurately, LOL is a fresh implementation of 1000Langs's goal, not a +rewrite of its code. Because the code was never Apache-2.0, there was no +licence switch for the code; the `meta/` data was never relicensed and is +still Apache-2.0. + +This PR does not edit the CHANGELOG. Its history entries are the owner's to +correct. + +== Consequences in this repository + +* `NOTICE` gives the Apache-2.0 attribution for `meta/`. +* `LICENSES/Apache-2.0.txt` is the 1000Langs `LICENSE`, byte for byte. +* `REUSE.toml` licenses `meta/**` as Apache-2.0. It credits Asgari + (2018-2019, the years of his `meta/` commits) only for the files he + compiled, and names no holder for the third-party captures and + unestablished files. +* Copyright lines that named Asgari on files he did not write now name + Jonathan D.A. Jewell. No licence identifier was changed. +* `README.adoc` credits 1000Langs as the inspiration and points here. diff --git a/proofs/POSTULATE-AUDIT.adoc b/proofs/POSTULATE-AUDIT.adoc index c75b927..936b8a6 100644 --- a/proofs/POSTULATE-AUDIT.adoc +++ b/proofs/POSTULATE-AUDIT.adoc @@ -112,8 +112,10 @@ layers of gate now prevent this class: == License note (unchanged) The Agda corpus files carry `MIT AND LicenseRef-Palimpsest-0.8` (not the -newer PMPL-1.0-or-later), reflecting attributed third-party origin -(Ehsaneddin Asgari, 2024). Acceptable as-is. +newer PMPL-1.0-or-later). Acceptable as-is. Their copyright lines formerly +named Ehsaneddin Asgari; that was a header error, not a provenance fact. The +corpus was written in this repository and has no counterpart in 1000Langs +(see docs/provenance/1000langs-comparison.adoc). == Remediation ledger diff --git a/proofs/README.adoc b/proofs/README.adoc index ca3be40..a904877 100644 --- a/proofs/README.adoc +++ b/proofs/README.adoc @@ -1,5 +1,5 @@ // SPDX-License-Identifier: CC-BY-SA-4.0 -// SPDX-FileCopyrightText: 2024-2026 Ehsaneddin Asgari and Contributors +// SPDX-FileCopyrightText: 2024-2026 Jonathan D.A. Jewell and Contributors = Mathematical Proofs for 1000Langs :toc: :toclevels: 3 diff --git a/proofs/echidna.toml b/proofs/echidna.toml index db4717c..e2ca2d4 100644 --- a/proofs/echidna.toml +++ b/proofs/echidna.toml @@ -1,5 +1,5 @@ # SPDX-License-Identifier: MPL-2.0 -# SPDX-FileCopyrightText: 2024-2025 Ehsaneddin Asgari and Contributors +# SPDX-FileCopyrightText: 2024-2026 Jonathan D.A. Jewell and Contributors # # Echidna Configuration for 1000Langs # ==================================== diff --git a/proofs/specs/alignment.spec b/proofs/specs/alignment.spec index 97a3e6e..664aeb2 100644 --- a/proofs/specs/alignment.spec +++ b/proofs/specs/alignment.spec @@ -1,5 +1,5 @@ -- SPDX-License-Identifier: MIT AND LicenseRef-Palimpsest-0.8 --- SPDX-FileCopyrightText: 2024-2025 Ehsaneddin Asgari and Contributors +-- SPDX-FileCopyrightText: 2024-2026 Jonathan D.A. Jewell and Contributors -- -- Parallel Corpus Alignment Specification -- ======================================== diff --git a/proofs/specs/iso639.spec b/proofs/specs/iso639.spec index 540fe05..da4aeaa 100644 --- a/proofs/specs/iso639.spec +++ b/proofs/specs/iso639.spec @@ -1,5 +1,5 @@ -- SPDX-License-Identifier: MIT AND LicenseRef-Palimpsest-0.8 --- SPDX-FileCopyrightText: 2024-2025 Ehsaneddin Asgari and Contributors +-- SPDX-FileCopyrightText: 2024-2026 Jonathan D.A. Jewell and Contributors -- -- ISO 639 Language Code Specification -- ==================================== diff --git a/proofs/specs/statistics.spec b/proofs/specs/statistics.spec index 5e8339a..0b79e2b 100644 --- a/proofs/specs/statistics.spec +++ b/proofs/specs/statistics.spec @@ -1,5 +1,5 @@ -- SPDX-License-Identifier: MIT AND LicenseRef-Palimpsest-0.8 --- SPDX-FileCopyrightText: 2024-2025 Ehsaneddin Asgari and Contributors +-- SPDX-FileCopyrightText: 2024-2026 Jonathan D.A. Jewell and Contributors -- -- Statistical Properties Specification -- ===================================== diff --git a/proofs/theories/InformationTheory.agda b/proofs/theories/InformationTheory.agda index 82f54ed..4c76844 100644 --- a/proofs/theories/InformationTheory.agda +++ b/proofs/theories/InformationTheory.agda @@ -1,5 +1,5 @@ -- SPDX-License-Identifier: MIT AND LicenseRef-Palimpsest-0.8 --- SPDX-FileCopyrightText: 2024-2026 Ehsaneddin Asgari and Contributors +-- SPDX-FileCopyrightText: 2024-2026 Jonathan D.A. Jewell and Contributors -- -- Information Theory — theorems -- ============================= diff --git a/proofs/theories/InformationTheory/Axioms.agda b/proofs/theories/InformationTheory/Axioms.agda index 3cf4215..76a6970 100644 --- a/proofs/theories/InformationTheory/Axioms.agda +++ b/proofs/theories/InformationTheory/Axioms.agda @@ -1,5 +1,5 @@ -- SPDX-License-Identifier: MIT AND LicenseRef-Palimpsest-0.8 --- SPDX-FileCopyrightText: 2024-2026 Ehsaneddin Asgari and Contributors +-- SPDX-FileCopyrightText: 2024-2026 Jonathan D.A. Jewell and Contributors -- -- Information Theory — quarantined trust base -- ============================================ diff --git a/proofs/theories/InformationTheory/Foundations.agda b/proofs/theories/InformationTheory/Foundations.agda index cf71c39..04ad215 100644 --- a/proofs/theories/InformationTheory/Foundations.agda +++ b/proofs/theories/InformationTheory/Foundations.agda @@ -1,5 +1,5 @@ -- SPDX-License-Identifier: MIT AND LicenseRef-Palimpsest-0.8 --- SPDX-FileCopyrightText: 2024-2026 Ehsaneddin Asgari and Contributors +-- SPDX-FileCopyrightText: 2024-2026 Jonathan D.A. Jewell and Contributors -- -- Information Theory — computational foundations -- ============================================== diff --git a/test/vitest.config.affine b/test/vitest.config.affine index 576473b..4a225be 100644 --- a/test/vitest.config.affine +++ b/test/vitest.config.affine @@ -4,7 +4,7 @@ module vitest.config; // SPDX-License-Identifier: MIT AND LicenseRef-Palimpsest-0.8 -// SPDX-FileCopyrightText: 2024-2025 Ehsaneddin Asgari and Contributors +// SPDX-FileCopyrightText: 2024-2026 Jonathan D.A. Jewell and Contributors import { defineConfig } from 'vitest/config'