diff --git a/docs/connections.md b/docs/connections.md index b6c0d482d09..ba37a079eaa 100644 --- a/docs/connections.md +++ b/docs/connections.md @@ -48,6 +48,25 @@ Syft uses Google Drive as its file-based transport layer. All peer-to-peer commu --- +## Encryption Policy + +Each client advertises an encryption policy in `SYFT_version.json`: + +| Policy | Set with | Effect | +| ---------- | -------------------------------------------- | ----------------------------------------- | +| `optional` | default on a new SyftBox, `encryption=False` | encrypt only with a peer that requires it | +| `required` | `encryption=True` | encrypt with every peer; refuse plaintext | + +A pair encrypts when one side is `required`. + +A SyftBox has one policy, kept in `//private/syft_settings.json`. The first client to start on a new SyftBox writes it: the `encryption` it logged in with, or `optional`. A later client, the notebook or syft-bg, logs in without `encryption` and gets the policy on disk. A client that asks for another policy raises `EncryptionPolicyMismatchError`, and so does a running client at its next sync when the file changed under it. Files at rest are read with the current policy, so to change the policy, delete the SyftBox (`client.delete_syftbox()`) and log in again. `syft-bg init --encryption ...` sets the policy the same way as a login. + +Nothing is sent to a peer until its policy is known, that is, until it accepts and shares its version file. A scientist's change for a peer whose policy or bundle is not known is dropped, and the send raises. An owner's send to that peer raises, and the change stays pending, so a later sync sends it. An owner keeps a message it cannot read yet in the inbox, with a warning. + +The format of a message must match the pair: plaintext in a pair that encrypts is refused, and so is an envelope in a pair that does not. The owner archives such a message unread, and a scientist skips it, each with a warning. + +A peer whose version file has no policy, written by a syft older than `PeerInfo`, counts as `optional`. A peer whose version file is not read yet has a policy that is not known: nothing is exchanged with it. + ## Peer Requests DS creates inbox/outbox folders on their own drive and grants the DO write access. The DO discovers these folders by searching for `syft_datasite#{my_email}#` folders they don't own. diff --git a/notebooks/ai_audit/internal/DO_Tutorial_V3.ipynb b/notebooks/ai_audit/internal/DO_Tutorial_V3.ipynb index bfb019d4fd9..f6f846222ba 100644 --- a/notebooks/ai_audit/internal/DO_Tutorial_V3.ipynb +++ b/notebooks/ai_audit/internal/DO_Tutorial_V3.ipynb @@ -858,6 +858,7 @@ "\n", "syft_bg.init(\n", " do_email=DO_EMAIL,\n", + " encryption=False,\n", " token_path=str(TOKEN_PATH),\n", " settings=settings,\n", ")\n" diff --git a/notebooks/ai_audit/internal/test_email_e2e.ipynb b/notebooks/ai_audit/internal/test_email_e2e.ipynb index 02ba57a88e7..9b5bc7e2d6b 100644 --- a/notebooks/ai_audit/internal/test_email_e2e.ipynb +++ b/notebooks/ai_audit/internal/test_email_e2e.ipynb @@ -195,6 +195,7 @@ "source": [ "syft_bg.init(\n", " do_email=DO_EMAIL,\n", + " encryption=False,\n", " token_path=do_token_path,\n", " settings={\n", " \"email_approve\": {\"gcp_project_id\": GCP_PROJECT_ID},\n", diff --git a/notebooks/ai_audit/internal/test_multiple_clients.ipynb b/notebooks/ai_audit/internal/test_multiple_clients.ipynb index e5bd55d4c0a..fe9597c7f92 100644 --- a/notebooks/ai_audit/internal/test_multiple_clients.ipynb +++ b/notebooks/ai_audit/internal/test_multiple_clients.ipynb @@ -228,6 +228,7 @@ "source": [ "syft_bg.init(\n", " do_email=DO_EMAIL,\n", + " encryption=False,\n", " token_path=str(DO_TOKEN),\n", " settings={\n", " \"email_approve\": {\"gcp_project_id\": GCP_PROJECT_ID},\n", @@ -363,6 +364,7 @@ "syft_bg.reset()\n", "syft_bg.init(\n", " do_email=DO_EMAIL,\n", + " encryption=False,\n", " token_path=str(DO_TOKEN),\n", " settings={\n", " \"email_approve\": {\"gcp_project_id\": GCP_PROJECT_ID},\n", diff --git a/notebooks/ai_audit/internal/test_syft_bg.ipynb b/notebooks/ai_audit/internal/test_syft_bg.ipynb index 12c78b367c9..2685a17c540 100644 --- a/notebooks/ai_audit/internal/test_syft_bg.ipynb +++ b/notebooks/ai_audit/internal/test_syft_bg.ipynb @@ -217,6 +217,7 @@ "source": [ "syft_bg.init(\n", " do_email=DO_EMAIL,\n", + " encryption=False,\n", " token_path=do_token_path,\n", " settings={\n", " \"email_approve\": {\"gcp_project_id\": GCP_PROJECT_ID},\n", @@ -282,7 +283,7 @@ "outputs": [], "source": [ "syft_bg.reset()\n", - "syft_bg.init(do_email=DO_EMAIL, token_path=do_token_path)\n", + "syft_bg.init(do_email=DO_EMAIL, encryption=False, token_path=do_token_path)\n", "\n", "syft_bg.ensure_running(services=[\"email_approve\"])\n", "time.sleep(3) # wait for subprocess to fail\n", @@ -309,7 +310,7 @@ "outputs": [], "source": [ "syft_bg.reset()\n", - "syft_bg.init(do_email=DO_EMAIL, token_path=do_token_path,\n", + "syft_bg.init(do_email=DO_EMAIL, encryption=False, token_path=do_token_path,\n", " settings={\"email_approve\": {\"gcp_project_id\": GCP_PROJECT_ID}})\n", "\n", "SERVICES = [\"sync\", \"notify\", \"approve\"]\n", @@ -447,6 +448,7 @@ "syft_bg.reset()\n", "syft_bg.init(\n", " do_email=DO_EMAIL,\n", + " encryption=False,\n", " token_path=do_token_path,\n", " settings={\n", " \"email_approve\": {\"gcp_project_id\": GCP_PROJECT_ID},\n", @@ -679,7 +681,7 @@ "bad_token = Path(tempfile.mktemp(suffix=\".json\"))\n", "bad_token.write_text(json.dumps({\"invalid\": True}))\n", "\n", - "syft_bg.init(do_email=DO_EMAIL, token_path=str(bad_token),\n", + "syft_bg.init(do_email=DO_EMAIL, encryption=False, token_path=str(bad_token),\n", " settings={\"email_approve\": {\"gcp_project_id\": GCP_PROJECT_ID}})\n", "\n", "syft_bg.ensure_running(services=[\"sync\"])\n", diff --git a/notebooks/enclave/double-blind-eval-w-receipts/1. DO-benchmark-owner-dbe.ipynb b/notebooks/enclave/double-blind-eval-w-receipts/1. DO-benchmark-owner-dbe.ipynb index a62cc6659ec..3793eaec9b1 100644 --- a/notebooks/enclave/double-blind-eval-w-receipts/1. DO-benchmark-owner-dbe.ipynb +++ b/notebooks/enclave/double-blind-eval-w-receipts/1. DO-benchmark-owner-dbe.ipynb @@ -102,13 +102,18 @@ "metadata": {}, "outputs": [], "source": [ - "CRYPTO_KEYS_PATH = None\n", - "# NOTE: Re-storing Private Key from local_file, if you have it, uncomment the following line and upload the file.\n", + "KEY_FILE = None\n", + "# To restore the encryption key of an earlier session, uncomment the next two lines and upload crypto_keys.json.\n", "# from google.colab import files; files.upload()\n", - "# CRYPTO_KEYS_PATH = \"crypto_keys.json\"\n", + "# KEY_FILE = \"crypto_keys.json\"\n", "\n", - "benchmark_owner = login_do(encryption=True, crypto_keys_path=CRYPTO_KEYS_PATH)\n", - "print(f\" Benchmark owner : {benchmark_owner.email}\")" + "benchmark_owner = login_do(key_file=KEY_FILE)\n", + "print(f\" Benchmark owner : {benchmark_owner.email}\")\n", + "\n", + "# First login: back up the encryption key. It lives only on this runtime's filesystem.\n", + "if KEY_FILE is None:\n", + " from google.colab import files\n", + " files.download(str(benchmark_owner.syftbox_folder / benchmark_owner.email / \"private\" / \"crypto_keys.json\"))" ] }, { @@ -119,7 +124,7 @@ "source": [ "# # Optionally, to start from a clean SyftBox\n", "# benchmark_owner.delete_syftbox()\n", - "# benchmark_owner._rds.peer_manager.write_own_version()" + "# benchmark_owner._rds.peer_manager.write_own_peer_info()" ] }, { diff --git a/notebooks/enclave/double-blind-eval-w-receipts/2. DO-model-owner-dbe.ipynb b/notebooks/enclave/double-blind-eval-w-receipts/2. DO-model-owner-dbe.ipynb index 9dcc0244ad4..52355a3d29c 100644 --- a/notebooks/enclave/double-blind-eval-w-receipts/2. DO-model-owner-dbe.ipynb +++ b/notebooks/enclave/double-blind-eval-w-receipts/2. DO-model-owner-dbe.ipynb @@ -100,13 +100,18 @@ "metadata": {}, "outputs": [], "source": [ - "CRYPTO_KEYS_PATH = None\n", - "# NOTE: Re-storing Private Key from local_file, if you have it, uncomment the following line and upload the file.\n", + "KEY_FILE = None\n", + "# To restore the encryption key of an earlier session, uncomment the next two lines and upload crypto_keys.json.\n", "# from google.colab import files; files.upload()\n", - "# CRYPTO_KEYS_PATH = \"crypto_keys.json\"\n", + "# KEY_FILE = \"crypto_keys.json\"\n", "\n", - "model_owner = login_do(encryption=True, crypto_keys_path=CRYPTO_KEYS_PATH)\n", - "print(f\" Model owner : {model_owner.email}\")" + "model_owner = login_do(key_file=KEY_FILE)\n", + "print(f\" Model owner : {model_owner.email}\")\n", + "\n", + "# First login: back up the encryption key. It lives only on this runtime's filesystem.\n", + "if KEY_FILE is None:\n", + " from google.colab import files\n", + " files.download(str(model_owner.syftbox_folder / model_owner.email / \"private\" / \"crypto_keys.json\"))" ] }, { @@ -117,7 +122,7 @@ "source": [ "# # Optionally, to start from a clean SyftBox\n", "# model_owner.delete_syftbox()\n", - "# model_owner._rds.peer_manager.write_own_version()" + "# model_owner._rds.peer_manager.write_own_peer_info()" ] }, { diff --git a/notebooks/enclave/gemma/colab-3-persona/1. DO-model-owner-gemma-restrict.ipynb b/notebooks/enclave/gemma/colab-3-persona/1. DO-model-owner-gemma-restrict.ipynb index a2dcf90cc9d..db58c2eae6f 100644 --- a/notebooks/enclave/gemma/colab-3-persona/1. DO-model-owner-gemma-restrict.ipynb +++ b/notebooks/enclave/gemma/colab-3-persona/1. DO-model-owner-gemma-restrict.ipynb @@ -125,7 +125,7 @@ "source": [ "# # Optionally to clear state\n", "# model_owner._manager.delete_syftbox()\n", - "# model_owner._manager.peer_manager.write_own_version()" + "# model_owner._manager.peer_manager.write_own_peer_info()" ] }, { diff --git a/notebooks/enclave/gemma/colab-3-persona/2. DO-benchmark-owner-gemma-restrict.ipynb b/notebooks/enclave/gemma/colab-3-persona/2. DO-benchmark-owner-gemma-restrict.ipynb index 58dbe0f80ac..b2e2e9353fd 100644 --- a/notebooks/enclave/gemma/colab-3-persona/2. DO-benchmark-owner-gemma-restrict.ipynb +++ b/notebooks/enclave/gemma/colab-3-persona/2. DO-benchmark-owner-gemma-restrict.ipynb @@ -98,7 +98,7 @@ "source": [ "# # Optionally to clear state\n", "# benchmark_owner._manager.delete_syftbox()\n", - "# benchmark_owner._manager.peer_manager.write_own_version()" + "# benchmark_owner._manager.peer_manager.write_own_peer_info()" ] }, { diff --git a/notebooks/enclave/gemma/colab-3-persona/3. DS-researcher-gemma-restrict.ipynb b/notebooks/enclave/gemma/colab-3-persona/3. DS-researcher-gemma-restrict.ipynb index c522f7a6a58..5511bdb7d54 100644 --- a/notebooks/enclave/gemma/colab-3-persona/3. DS-researcher-gemma-restrict.ipynb +++ b/notebooks/enclave/gemma/colab-3-persona/3. DS-researcher-gemma-restrict.ipynb @@ -96,7 +96,7 @@ "source": [ "# # Optionally to clear state\n", "# researcher._manager.delete_syftbox()\n", - "# researcher._manager.peer_manager.write_own_version()" + "# researcher._manager.peer_manager.write_own_peer_info()" ] }, { diff --git a/notebooks/enclave/gemma/colab-shield/0. clear-state-gemma.ipynb b/notebooks/enclave/gemma/colab-shield/0. clear-state-gemma.ipynb index e04ad0f1990..05ebd26fd2f 100644 --- a/notebooks/enclave/gemma/colab-shield/0. clear-state-gemma.ipynb +++ b/notebooks/enclave/gemma/colab-shield/0. clear-state-gemma.ipynb @@ -12,7 +12,7 @@ "\n", "```python\n", "client._manager.delete_syftbox()\n", - "client._manager.peer_manager.write_own_version()\n", + "client._manager.peer_manager.write_own_peer_info()\n", "```" ] }, @@ -68,7 +68,7 @@ "\n", "client = login_do(sync=False, load_peers=False)\n", "client._rds.delete_syftbox()\n", - "client._rds.peer_manager.write_own_version()" + "client._rds.peer_manager.write_own_peer_info()" ] } ], diff --git a/notebooks/enclave/gemma/colab-shield/1. DO-model-owner-gemma-shield.ipynb b/notebooks/enclave/gemma/colab-shield/1. DO-model-owner-gemma-shield.ipynb index d5940be0c86..3b24c97f59b 100644 --- a/notebooks/enclave/gemma/colab-shield/1. DO-model-owner-gemma-shield.ipynb +++ b/notebooks/enclave/gemma/colab-shield/1. DO-model-owner-gemma-shield.ipynb @@ -156,16 +156,16 @@ "metadata": {}, "outputs": [], "source": [ - "CRYPTO_KEYS_PATH = None\n", - "# NOTE: Re-storing Private Key from local_file, if you have it, uncomment the following line and upload the file.\n", + "KEY_FILE = None\n", + "# To restore the encryption key of an earlier session, uncomment the next two lines and upload crypto_keys.json.\n", "# from google.colab import files; files.upload()\n", - "# CRYPTO_KEYS_PATH = \"crypto_keys.json\"\n", + "# KEY_FILE = \"crypto_keys.json\"\n", "\n", - "model_owner = login_do(encryption=True, crypto_keys_path=CRYPTO_KEYS_PATH)\n", + "model_owner = login_do(key_file=KEY_FILE)\n", "print(f\" Model owner : {model_owner.email}\")\n", "\n", - "# First login: back up the encryption key — it lives only on this runtime's filesystem.\n", - "if CRYPTO_KEYS_PATH is None:\n", + "# First login: back up the encryption key. It lives only on this runtime's filesystem.\n", + "if KEY_FILE is None:\n", " from google.colab import files\n", " files.download(str(model_owner.syftbox_folder / model_owner.email / \"private\" / \"crypto_keys.json\"))" ] @@ -179,7 +179,7 @@ "source": [ "# # Optionally to clear state\n", "# model_owner._manager.delete_syftbox()\n", - "# model_owner._manager.peer_manager.write_own_version()" + "# model_owner._manager.peer_manager.write_own_peer_info()" ] }, { diff --git a/notebooks/enclave/gemma/colab-shield/2. DO-benchmark-owner-gemma-shield.ipynb b/notebooks/enclave/gemma/colab-shield/2. DO-benchmark-owner-gemma-shield.ipynb index 04d449d4f66..b59906b7c26 100644 --- a/notebooks/enclave/gemma/colab-shield/2. DO-benchmark-owner-gemma-shield.ipynb +++ b/notebooks/enclave/gemma/colab-shield/2. DO-benchmark-owner-gemma-shield.ipynb @@ -100,16 +100,16 @@ "metadata": {}, "outputs": [], "source": [ - "CRYPTO_KEYS_PATH = None\n", - "# NOTE: Re-storing Private Key from local_file, if you have it, uncomment the following line and upload the file.\n", + "KEY_FILE = None\n", + "# To restore the encryption key of an earlier session, uncomment the next two lines and upload crypto_keys.json.\n", "# from google.colab import files; files.upload()\n", - "# CRYPTO_KEYS_PATH = \"crypto_keys.json\"\n", + "# KEY_FILE = \"crypto_keys.json\"\n", "\n", - "benchmark_owner = login_do(encryption=True, crypto_keys_path=CRYPTO_KEYS_PATH)\n", + "benchmark_owner = login_do(key_file=KEY_FILE)\n", "print(f\" Benchmark owner : {benchmark_owner.email}\")\n", "\n", - "# First login: back up the encryption key — it lives only on this runtime's filesystem.\n", - "if CRYPTO_KEYS_PATH is None:\n", + "# First login: back up the encryption key. It lives only on this runtime's filesystem.\n", + "if KEY_FILE is None:\n", " from google.colab import files\n", " files.download(str(benchmark_owner.syftbox_folder / benchmark_owner.email / \"private\" / \"crypto_keys.json\"))" ] @@ -123,7 +123,7 @@ "source": [ "# # Optionally to clear state\n", "# benchmark_owner._manager.delete_syftbox()\n", - "# benchmark_owner._manager.peer_manager.write_own_version()" + "# benchmark_owner._manager.peer_manager.write_own_peer_info()" ] }, { diff --git a/notebooks/enclave/gemma/colab/0. clear-state-gemma.ipynb b/notebooks/enclave/gemma/colab/0. clear-state-gemma.ipynb index 483e633c75c..d351ed54a83 100644 --- a/notebooks/enclave/gemma/colab/0. clear-state-gemma.ipynb +++ b/notebooks/enclave/gemma/colab/0. clear-state-gemma.ipynb @@ -12,7 +12,7 @@ "\n", "```python\n", "client._manager.delete_syftbox()\n", - "client._manager.peer_manager.write_own_version()\n", + "client._manager.peer_manager.write_own_peer_info()\n", "```" ] }, @@ -68,7 +68,7 @@ "\n", "client = login_do(sync=False, load_peers=False)\n", "client._manager.delete_syftbox()\n", - "client._manager.peer_manager.write_own_version()" + "client._manager.peer_manager.write_own_peer_info()" ] } ], diff --git a/notebooks/enclave/gemma/colab/1. DO-model-owner-gemma-restrict.ipynb b/notebooks/enclave/gemma/colab/1. DO-model-owner-gemma-restrict.ipynb index 7172f738261..ce7795c9d1f 100644 --- a/notebooks/enclave/gemma/colab/1. DO-model-owner-gemma-restrict.ipynb +++ b/notebooks/enclave/gemma/colab/1. DO-model-owner-gemma-restrict.ipynb @@ -108,16 +108,16 @@ "metadata": {}, "outputs": [], "source": [ - "CRYPTO_KEYS_PATH = None\n", - "# NOTE: Re-storing Private Key from local_file, if you have it, uncomment the following line and upload the file.\n", + "KEY_FILE = None\n", + "# To restore the encryption key of an earlier session, uncomment the next two lines and upload crypto_keys.json.\n", "# from google.colab import files; files.upload()\n", - "# CRYPTO_KEYS_PATH = \"crypto_keys.json\"\n", + "# KEY_FILE = \"crypto_keys.json\"\n", "\n", - "model_owner = login_do(encryption=True, crypto_keys_path=CRYPTO_KEYS_PATH)\n", + "model_owner = login_do(key_file=KEY_FILE)\n", "print(f\" Model owner : {model_owner.email}\")\n", "\n", - "# First login: back up the encryption key — it lives only on this runtime's filesystem.\n", - "if CRYPTO_KEYS_PATH is None:\n", + "# First login: back up the encryption key. It lives only on this runtime's filesystem.\n", + "if KEY_FILE is None:\n", " from google.colab import files\n", " files.download(str(model_owner.syftbox_folder / model_owner.email / \"private\" / \"crypto_keys.json\"))" ] @@ -131,7 +131,7 @@ "source": [ "# # Optionally to clear state\n", "# model_owner._manager.delete_syftbox()\n", - "# model_owner._manager.peer_manager.write_own_version()" + "# model_owner._manager.peer_manager.write_own_peer_info()" ] }, { diff --git a/notebooks/enclave/gemma/colab/2. DO-benchmark-owner-gemma-restrict.ipynb b/notebooks/enclave/gemma/colab/2. DO-benchmark-owner-gemma-restrict.ipynb index 6ab0d7f817a..183cf35a166 100644 --- a/notebooks/enclave/gemma/colab/2. DO-benchmark-owner-gemma-restrict.ipynb +++ b/notebooks/enclave/gemma/colab/2. DO-benchmark-owner-gemma-restrict.ipynb @@ -89,16 +89,16 @@ "metadata": {}, "outputs": [], "source": [ - "CRYPTO_KEYS_PATH = None\n", - "# NOTE: Re-storing Private Key from local_file, if you have it, uncomment the following line and upload the file.\n", + "KEY_FILE = None\n", + "# To restore the encryption key of an earlier session, uncomment the next two lines and upload crypto_keys.json.\n", "# from google.colab import files; files.upload()\n", - "# CRYPTO_KEYS_PATH = \"crypto_keys.json\"\n", + "# KEY_FILE = \"crypto_keys.json\"\n", "\n", - "benchmark_owner = login_do(encryption=True, crypto_keys_path=CRYPTO_KEYS_PATH)\n", + "benchmark_owner = login_do(key_file=KEY_FILE)\n", "print(f\" Benchmark owner : {benchmark_owner.email}\")\n", "\n", - "# First login: back up the encryption key — it lives only on this runtime's filesystem.\n", - "if CRYPTO_KEYS_PATH is None:\n", + "# First login: back up the encryption key. It lives only on this runtime's filesystem.\n", + "if KEY_FILE is None:\n", " from google.colab import files\n", " files.download(str(benchmark_owner.syftbox_folder / benchmark_owner.email / \"private\" / \"crypto_keys.json\"))" ] @@ -112,7 +112,7 @@ "source": [ "# # Optionally to clear state\n", "# benchmark_owner._manager.delete_syftbox()\n", - "# benchmark_owner._manager.peer_manager.write_own_version()" + "# benchmark_owner._manager.peer_manager.write_own_peer_info()" ] }, { diff --git a/notebooks/enclave/nanolm/2. enclave_eval_localenc.ipynb b/notebooks/enclave/nanolm/2. enclave_eval_localenc.ipynb index 054860e96dc..01b000e01b3 100644 --- a/notebooks/enclave/nanolm/2. enclave_eval_localenc.ipynb +++ b/notebooks/enclave/nanolm/2. enclave_eval_localenc.ipynb @@ -272,9 +272,9 @@ "researcher._manager.delete_syftbox()\n", "\n", "# Write Own Version\n", - "model_owner._manager.peer_manager.write_own_version()\n", - "benchmark_owner._manager.peer_manager.write_own_version()\n", - "researcher._manager.peer_manager.write_own_version()" + "model_owner._manager.peer_manager.write_own_peer_info()\n", + "benchmark_owner._manager.peer_manager.write_own_peer_info()\n", + "researcher._manager.peer_manager.write_own_peer_info()" ] }, { diff --git a/notebooks/enclave/nanolm/3. enclave_eval_e2e.ipynb b/notebooks/enclave/nanolm/3. enclave_eval_e2e.ipynb index fc24f7c314d..f28dfb7e10f 100644 --- a/notebooks/enclave/nanolm/3. enclave_eval_e2e.ipynb +++ b/notebooks/enclave/nanolm/3. enclave_eval_e2e.ipynb @@ -177,9 +177,9 @@ "metadata": {}, "outputs": [], "source": [ - "# model_owner._manager.peer_manager.write_own_version()\n", - "# benchmark_owner._manager.peer_manager.write_own_version()\n", - "# researcher._manager.peer_manager.write_own_version()" + "# model_owner._manager.peer_manager.write_own_peer_info()\n", + "# benchmark_owner._manager.peer_manager.write_own_peer_info()\n", + "# researcher._manager.peer_manager.write_own_peer_info()" ] }, { diff --git a/notebooks/enclave/nanolm/nbsplit/1. DO-model-owner-nanolm.ipynb b/notebooks/enclave/nanolm/nbsplit/1. DO-model-owner-nanolm.ipynb index d3e9923d1e2..a48200654ac 100644 --- a/notebooks/enclave/nanolm/nbsplit/1. DO-model-owner-nanolm.ipynb +++ b/notebooks/enclave/nanolm/nbsplit/1. DO-model-owner-nanolm.ipynb @@ -95,7 +95,7 @@ "source": [ "# Optionally to clear state\n", "model_owner._manager.delete_syftbox()\n", - "model_owner._manager.peer_manager.write_own_version()" + "model_owner._manager.peer_manager.write_own_peer_info()" ] }, { diff --git a/notebooks/enclave/nanolm/nbsplit/2. DO-benchmark-owner-nanolm.ipynb b/notebooks/enclave/nanolm/nbsplit/2. DO-benchmark-owner-nanolm.ipynb index 14e6987118f..9750daf86f0 100644 --- a/notebooks/enclave/nanolm/nbsplit/2. DO-benchmark-owner-nanolm.ipynb +++ b/notebooks/enclave/nanolm/nbsplit/2. DO-benchmark-owner-nanolm.ipynb @@ -95,7 +95,7 @@ "source": [ "# Optionally to clear state\n", "benchmark_owner._manager.delete_syftbox()\n", - "benchmark_owner._manager.peer_manager.write_own_version()" + "benchmark_owner._manager.peer_manager.write_own_peer_info()" ] }, { diff --git a/notebooks/enclave/nanolm/nbsplit/3. DS-researcher-nanolm.ipynb b/notebooks/enclave/nanolm/nbsplit/3. DS-researcher-nanolm.ipynb index 45caa9c9f5e..760c795fdc2 100644 --- a/notebooks/enclave/nanolm/nbsplit/3. DS-researcher-nanolm.ipynb +++ b/notebooks/enclave/nanolm/nbsplit/3. DS-researcher-nanolm.ipynb @@ -96,7 +96,7 @@ "source": [ "# Optionally to clear state\n", "researcher._manager.delete_syftbox()\n", - "researcher._manager.peer_manager.write_own_version()" + "researcher._manager.peer_manager.write_own_peer_info()" ] }, { diff --git a/packages/enclave-model-api-example/docker/inference_server.py b/packages/enclave-model-api-example/docker/inference_server.py index cad573a4b60..b73b6d8c1c5 100644 --- a/packages/enclave-model-api-example/docker/inference_server.py +++ b/packages/enclave-model-api-example/docker/inference_server.py @@ -48,4 +48,4 @@ ), ) service.start_polling() -app.include_router(build_router(service, use_encryption=settings.use_encryption)) +app.include_router(build_router(service)) diff --git a/packages/enclave-model-api-example/notebooks/demo.ipynb b/packages/enclave-model-api-example/notebooks/demo.ipynb index a4a72d8d511..7ad565a81df 100644 --- a/packages/enclave-model-api-example/notebooks/demo.ipynb +++ b/packages/enclave-model-api-example/notebooks/demo.ipynb @@ -36,8 +36,8 @@ "import os\n", "from pathlib import Path\n", "\n", - "# Deployed PROD enclave (real Gemma model, encryption ON). Encryption is auto-detected\n", - "# from /model-status in the next cell, so no ENCLAVE_ENCRYPTION env var is needed.\n", + "# Deployed PROD enclave (real Gemma model, encryption ON). The data owners encrypt\n", + "# because the enclave policy requires it, so they need no encryption setting.\n", "os.environ[\"ENCLAVE_URL\"] = \"http://34.68.46.255:8080\"\n", "os.environ[\"GEMMA_WEIGHTS_DIR\"] = (\n", " f\"{Path.home().resolve()}/.cache/kagglehub/models/google/gemma-3/flax/gemma-3-270m-it/1\"\n", @@ -65,15 +65,6 @@ "# deployed VM, e.g. \"http://:8080\" — defaults to a local server if unset.\n", "ENCLAVE_URL = os.environ.get(\"ENCLAVE_URL\", \"http://localhost:8080\")\n", "\n", - "# Auto-match the enclave's encryption setting: data owners MUST use the same\n", - "# encryption mode as the enclave or Drive sync silently fails. The enclave\n", - "# reports it via /model-status (\"use_encryption\"); fall back to the\n", - "# ENCLAVE_ENCRYPTION env var if the endpoint doesn't expose it (older image).\n", - "_status = requests.get(f\"{ENCLAVE_URL}/model-status\").json()\n", - "ENCRYPTION = _status.get(\"use_encryption\")\n", - "if ENCRYPTION is None:\n", - " ENCRYPTION = os.environ.get(\"ENCLAVE_ENCRYPTION\", \"false\").lower() == \"true\"\n", - "\n", "# Local dir holding the real Gemma 3 270m flax weights that DO1 uploads. Expected layout:\n", "# /tokenizer.model and //\n", "WEIGHTS_DIR = Path(\n", @@ -82,7 +73,7 @@ " \"~/.cache/kagglehub/models/google/gemma-3/flax/gemma-3-270m-it/1\",\n", " )\n", ").expanduser()\n", - "ENCLAVE_URL, ENCRYPTION, WEIGHTS_DIR" + "ENCLAVE_URL, WEIGHTS_DIR" ] }, { @@ -111,10 +102,8 @@ "metadata": {}, "outputs": [], "source": [ - "do1 = login_do(DO1, CRED / \"token_do.json\", encryption=ENCRYPTION) # model + log owner\n", - "do2 = login_do(\n", - " DO2, CRED / \"token_ds.json\", encryption=ENCRYPTION\n", - ") # submits the analysis job" + "do1 = login_do(DO1, CRED / \"token_do.json\") # model + log owner\n", + "do2 = login_do(DO2, CRED / \"token_ds.json\") # submits the analysis job" ] }, { diff --git a/packages/enclave-model-api-example/scripts/local_server.py b/packages/enclave-model-api-example/scripts/local_server.py index b5676054772..a1fec698822 100644 --- a/packages/enclave-model-api-example/scripts/local_server.py +++ b/packages/enclave-model-api-example/scripts/local_server.py @@ -46,7 +46,7 @@ ) service.start_polling() uvicorn.run( - create_app(service, use_encryption=settings.use_encryption), + create_app(service), host="0.0.0.0", port=8080, ) diff --git a/packages/enclave-model-api-example/scripts/run_demo.py b/packages/enclave-model-api-example/scripts/run_demo.py index 71602baedb0..9e47be3b7d5 100644 --- a/packages/enclave-model-api-example/scripts/run_demo.py +++ b/packages/enclave-model-api-example/scripts/run_demo.py @@ -62,10 +62,6 @@ ) ).expanduser() EXPECT_MODEL = os.environ.get("DEMO_EXPECT_MODEL", "auto").lower() # auto|real|mock -# Encrypt all Drive communication. MUST match the enclave: the prod `inference-start` -# recipe forces SYFT_ENCLAVE_USE_ENCRYPTION=true, so set DEMO_ENCRYPTION=true when -# running against a prod enclave (debug VMs default to encryption off). -ENCRYPTION = os.environ.get("DEMO_ENCRYPTION", "false").lower() in ("1", "true", "yes") ENCLAVE = "beach.do.008@gmail.com" DO1 = "koenlennartvanderveen@gmail.com" # model + log owner @@ -201,15 +197,10 @@ def main(): "DEMO_EXPECT_MODEL=mock but the enclave is serving the REAL backend." ) print(f"enclave backend: {'REAL Gemma' if server_real else 'mock'}") - # Match the enclave's encryption setting (reported by /model-status). Fall - # back to DEMO_ENCRYPTION if the endpoint doesn't expose it (older image). - server_enc = status.get("use_encryption") - encryption = ENCRYPTION if server_enc is None else bool(server_enc) - + # The data owners encrypt with the enclave when its policy requires it. banner("log in the two data owners") - print(f"encryption: {encryption}") - do1 = login_do(DO1, CRED / "token_do.json", encryption=encryption) - do2 = login_do(DO2, CRED / "token_ds.json", encryption=encryption) + do1 = login_do(DO1, CRED / "token_do.json") + do2 = login_do(DO2, CRED / "token_ds.json") banner("peer with the enclave, wait until connected") do1.add_peer(ENCLAVE) diff --git a/packages/enclave-model-api-example/src/enclave_model_api/__main__.py b/packages/enclave-model-api-example/src/enclave_model_api/__main__.py index b8b307b2967..e3a4c42d04e 100644 --- a/packages/enclave-model-api-example/src/enclave_model_api/__main__.py +++ b/packages/enclave-model-api-example/src/enclave_model_api/__main__.py @@ -48,7 +48,6 @@ def main() -> None: f"Enclave settings — email={settings.email} data_owners={settings.data_owners} " f"token_path={settings.token_path} poll_interval={settings.poll_interval}s " f"require_tee={settings.require_tee} fresh_state={settings.fresh_state} " - f"use_encryption={settings.use_encryption} " f"persist_owner_state={settings.persist_owner_state}" ) logger.info( @@ -62,7 +61,6 @@ def main() -> None: email=settings.email, token_path=settings.token_path, data_owners=settings.data_owners, - encryption=settings.use_encryption, persist_owner_state=settings.persist_owner_state, ) logger.info("SyftEnclaveClient ready") diff --git a/packages/enclave-model-api-example/src/enclave_model_api/server.py b/packages/enclave-model-api-example/src/enclave_model_api/server.py index e4cc0d86983..ff0373a35a9 100644 --- a/packages/enclave-model-api-example/src/enclave_model_api/server.py +++ b/packages/enclave-model-api-example/src/enclave_model_api/server.py @@ -20,9 +20,7 @@ class InferResponse(BaseModel): stats: dict -def build_router( - service: InferenceService, use_encryption: bool | None = None -) -> APIRouter: +def build_router(service: InferenceService) -> APIRouter: router = APIRouter() @router.post("/infer", response_model=InferResponse) @@ -44,21 +42,15 @@ def model_status() -> dict: "mock": not getattr(service.backend, "requires_weights", True), "weights_present": service.weights_present, "model_loaded": service.loaded, - # Whether the enclave encrypts all Drive communication. Data owners - # must use the SAME setting or peering/sync silently fails, so the - # notebook reads this to match login_do(encryption=...) automatically. - "use_encryption": use_encryption, } return router -def create_app( - service: InferenceService, use_encryption: bool | None = None -) -> FastAPI: +def create_app(service: InferenceService) -> FastAPI: """Standalone app for tests and local runs (docker combines with attestation).""" app = FastAPI(title="Syft Enclave Inference", version="0.1.0") - app.include_router(build_router(service, use_encryption=use_encryption)) + app.include_router(build_router(service)) @app.get("/health") def health() -> dict: diff --git a/packages/enclave-model-api-example/tests/test_inference_server.py b/packages/enclave-model-api-example/tests/test_inference_server.py index 5866ea319de..062617df0dc 100644 --- a/packages/enclave-model-api-example/tests/test_inference_server.py +++ b/packages/enclave-model-api-example/tests/test_inference_server.py @@ -104,7 +104,7 @@ def test_inference_server_full_lifecycle(tmp_path): weights_dir=weights_dir, logs_dir=logs_dir, ) - client = TestClient(create_app(service, use_encryption=True)) + client = TestClient(create_app(service)) # Weights not synced yet status = client.get("/model-status").json() @@ -113,7 +113,6 @@ def test_inference_server_full_lifecycle(tmp_path): "mock": False, "weights_present": False, "model_loaded": False, - "use_encryption": True, } assert client.post("/infer", json={"query": "hi"}).status_code == 503 diff --git a/packages/syft-bg/docs/python-api.md b/packages/syft-bg/docs/python-api.md index b9dea4a74e5..e874e284ad6 100644 --- a/packages/syft-bg/docs/python-api.md +++ b/packages/syft-bg/docs/python-api.md @@ -7,11 +7,15 @@ import syft_bg ## Init ```python -syft_bg.init(do_email="you@example.com") +syft_bg.init(do_email="you@example.com", encryption=False) ``` Creates the config file. Optionally pass `syftbox_root` and `token_path`. +`encryption` is required. Pass the same value as the notebook's `login_do` +(`True` or `False`). syft-bg writes the same datasite as the +notebook, so the two must use the same policy. + ## Ensure running ```python @@ -98,7 +102,7 @@ syft_bg.logs("notify") # last 50 lines of notify service l import syft_bg # Create config -syft_bg.init(do_email="you@example.com") +syft_bg.init(do_email="you@example.com", encryption=False) # Start services syft_bg.ensure_running(["notify", "approve"]) diff --git a/packages/syft-bg/src/syft_bg/api/api.py b/packages/syft-bg/src/syft_bg/api/api.py index d1e53e39604..9e1504b8873 100644 --- a/packages/syft-bg/src/syft_bg/api/api.py +++ b/packages/syft-bg/src/syft_bg/api/api.py @@ -10,6 +10,12 @@ PolicyRunner, PolicyState, ) +from syft_rds import ( + EncryptionPolicy, + datasite_settings_path, + get_jupyter_default_syftbox_folder, + resolve_encryption_policy, +) from syft_rds.apis import ApiArg from syft_bg.api.results import ApiResult, InstallationResult, StatusResult @@ -52,10 +58,26 @@ def init( do_email: str, + encryption: bool | None = None, syftbox_root: str | Path | None = None, token_path: str | Path | None = None, settings: dict[str, dict] | None = None, ) -> None: + """Write the syft-bg config for ``do_email``. + + ``encryption`` takes the same values as ``login_do``. The policy is kept + in the SyftBox, not in the syft-bg config, so syft-bg and the notebook + share it. Without ``encryption``, the SyftBox keeps its policy. + + Raises: + EncryptionPolicyMismatchError: if ``encryption`` differs from the + policy of the SyftBox. + """ + root = syftbox_root or get_jupyter_default_syftbox_folder(do_email) + resolve_encryption_policy( + datasite_settings_path(root, do_email), EncryptionPolicy.from_bool(encryption) + ) + token_kwargs: dict = {} if token_path is not None: # A single OAuth token is used for both the Gmail and Drive APIs. diff --git a/packages/syft-bg/src/syft_bg/cli/commands.py b/packages/syft-bg/src/syft_bg/cli/commands.py index ca6d9320b46..84989da00e3 100644 --- a/packages/syft-bg/src/syft_bg/cli/commands.py +++ b/packages/syft-bg/src/syft_bg/cli/commands.py @@ -238,7 +238,19 @@ def run_foreground(service: str, once: bool): type=click.Path(exists=True), help="Path to the OAuth token file.", ) -def init(email: str, syftbox_root: str | None, token_path: str | None): +@click.option( + "--encryption", + type=click.Choice(["optional", "required"]), + default=None, + help="Encryption policy of the SyftBox. Without it, the SyftBox keeps its " + "policy, or uses optional when it has none.", +) +def init( + email: str, + syftbox_root: str | None, + token_path: str | None, + encryption: str | None, +): """Initialize syft-bg configuration. Sets up the config file with the data owner email and optional @@ -248,7 +260,7 @@ def init(email: str, syftbox_root: str | None, token_path: str | None): syft-bg init -e alice@uni.edu - syft-bg init -e alice@uni.edu -r ~/syftbox -t ~/token.json + syft-bg init -e alice@uni.edu --encryption required -r ~/syftbox -t ~/token.json """ from syft_bg.api.api import init as api_init @@ -256,6 +268,7 @@ def init(email: str, syftbox_root: str | None, token_path: str | None): do_email=email, syftbox_root=syftbox_root, token_path=token_path, + encryption=None if encryption is None else encryption == "required", ) diff --git a/packages/syft-bg/src/syft_bg/cli/init/flow.py b/packages/syft-bg/src/syft_bg/cli/init/flow.py index 253c49d048c..a218595fef8 100644 --- a/packages/syft-bg/src/syft_bg/cli/init/flow.py +++ b/packages/syft-bg/src/syft_bg/cli/init/flow.py @@ -7,6 +7,13 @@ from syft_bg.cli.init.exceptions import InitFlowError from syft_bg.common.config import get_default_paths +from syft_rds import ( + EncryptionPolicy, + EncryptionPolicyMismatchError, + datasite_settings_path, + resolve_encryption_policy, +) + from syft_bg.common.syft_bg_config import SyftBgConfig @@ -21,6 +28,7 @@ class UserPassedConfig: # Core settings email: str | None = None syftbox_root: str | None = None + encryption: str | None = None # Control flags yes: bool = False # Auto-confirm config update @@ -77,11 +85,12 @@ def _load_existing_config(config: UserPassedConfig, config_path: Path) -> SyftBg if not config.quiet: click.echo(f"Found existing config at {config_path}") - return SyftBgConfig.from_path(config_path) + # Without the merge, so the services keep inheriting the top-level fields. + return SyftBgConfig.from_path(config_path, merge=False) def _resolve_common_settings(config: UserPassedConfig, result: SyftBgConfig) -> None: - """Resolve email and syftbox_root, mutating result.""" + """Resolve email, syftbox_root and the encryption policy, mutating result.""" default_email = result.do_email or "" if config.email is not None: @@ -116,6 +125,32 @@ def _resolve_common_settings(config: UserPassedConfig, result: SyftBgConfig) -> "SyftBox root directory", default=default_syftbox ) + _resolve_encryption(config, result) + + +def _resolve_encryption(config: UserPassedConfig, result: SyftBgConfig) -> None: + """Set the encryption policy of the SyftBox, or keep the one it has. + + The policy lives in the SyftBox, so syft-bg and the notebook share it. + Only a new SyftBox prompts for it. + """ + settings_path = datasite_settings_path(result.syftbox_root, result.do_email) + requested = EncryptionPolicy(config.encryption) if config.encryption else None + if requested is None and not config.quiet and not settings_path.exists(): + requested = EncryptionPolicy( + click.prompt( + "Encryption policy", + type=click.Choice([p.value for p in EncryptionPolicy]), + default=EncryptionPolicy.OPTIONAL.value, + ) + ) + try: + policy = resolve_encryption_policy(settings_path, requested) + except EncryptionPolicyMismatchError as e: + raise InitFlowError(str(e)) from e + if not config.quiet: + click.echo(f"Encryption policy: {policy.value}") + # def _setup_auth( # user_passed_config: UserPassedConfig, diff --git a/packages/syft-datasets/scripts/generate_release_fixture.py b/packages/syft-datasets/scripts/generate_release_fixture.py index f21d9c20487..4d05680245c 100644 --- a/packages/syft-datasets/scripts/generate_release_fixture.py +++ b/packages/syft-datasets/scripts/generate_release_fixture.py @@ -41,9 +41,9 @@ def create_syftbox_with_datasets_created_by_manager( """One dataset written in the given protocol layout (mock + private + readme).""" mgr = SyftDatasetManager(syftbox_folder_path=syftbox, email=DO_EMAIL) # Force the target protocol by advertising a peer at that version. - mgr.storage.peer_schemas = { + mgr.storage.peer_schema = { DS_EMAIL: dataset_registry.schema_for_protocol_version(protocol_version) - } + }.get mock_path = src_dir / "mock.csv" mock_path.write_text("id,value\n1,10\n2,20\n") diff --git a/packages/syft-datasets/src/syft_datasets/dataset_manager.py b/packages/syft-datasets/src/syft_datasets/dataset_manager.py index ccd2311357d..7c175ff50bf 100644 --- a/packages/syft-datasets/src/syft_datasets/dataset_manager.py +++ b/packages/syft-datasets/src/syft_datasets/dataset_manager.py @@ -1,4 +1,5 @@ from pathlib import Path +from typing import Callable from typing_extensions import Self @@ -27,29 +28,28 @@ def __init__( self, syftbox_folder_path: PathLike, email: str, - peer_schemas: dict[str, ProtocolSchema] | None = None, + peer_schema: Callable[[str], ProtocolSchema | None] | None = None, ): self.syftbox_config = SyftBoxConfig( syftbox_folder=to_path(syftbox_folder_path), email=email ) - # peer_schemas (peer email -> dataset ProtocolSchema): syft - # passes PeerManager's live map here (updated in place as peer version - # files load). A named peer without an entry resolves to the floor, so - # the dataset stays readable by a peer of unknown version. + # peer_schema (peer email -> dataset ProtocolSchema) comes from syft. A + # named peer without a schema resolves to the floor, so the dataset + # stays readable by a peer of unknown version. self.storage = DatasetStorage( - config=self.syftbox_config, peer_schemas=peer_schemas + config=self.syftbox_config, peer_schema=peer_schema ) @classmethod def from_config( cls, config: SyftBoxConfig, - peer_schemas: dict[str, ProtocolSchema] | None = None, + peer_schema: Callable[[str], ProtocolSchema | None] | None = None, ) -> Self: return cls( syftbox_folder_path=config.syftbox_folder, email=config.email, - peer_schemas=peer_schemas, + peer_schema=peer_schema, ) def create( diff --git a/packages/syft-datasets/src/syft_datasets/dataset_storage.py b/packages/syft-datasets/src/syft_datasets/dataset_storage.py index 11fbc07292a..a5531b6b3c6 100644 --- a/packages/syft-datasets/src/syft_datasets/dataset_storage.py +++ b/packages/syft-datasets/src/syft_datasets/dataset_storage.py @@ -3,7 +3,7 @@ from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path -from typing import Iterator, Optional +from typing import Callable, Iterator, Optional from uuid import UUID, uuid4 from syft_migration import ( @@ -49,6 +49,11 @@ def _utcnow() -> datetime: return datetime.now(tz=timezone.utc) +def _no_peer_schema(peer_email: str) -> Optional[ProtocolSchema]: + """No peer schema is known: the default when syft passes no lookup.""" + return None + + @dataclass(frozen=True) class DatasetSourceFiles: """Source paths a caller hands to ``create_dataset`` (ephemeral input). @@ -96,20 +101,16 @@ def __init__( self, config: SyftBoxConfig, registry: MigrationRegistry = dataset_registry, - peer_schemas: Optional[dict[str, ProtocolSchema]] = None, + peer_schema: Optional[Callable[[str], Optional[ProtocolSchema]]] = None, ) -> None: self.config = config self.registry = registry self.service = MigrationService(registry=registry) - # peer email -> dataset ProtocolSchema; syft passes PeerManager's - # live map here (updated in place as peer version files load). A named - # peer without an entry cannot be assumed to read the current layout, so - # it resolves to the floor. - # `is not None`, not `or`: the live dict starts empty and `or {}` would - # drop the shared reference, freezing negotiation at construction time. - self.peer_schemas: dict[str, ProtocolSchema] = ( - peer_schemas if peer_schemas is not None else {} - ) + # peer email -> the dataset ProtocolSchema it advertises, or None when + # not known; syft looks it up in the peer store on each call. A named + # peer without a schema cannot be assumed to read the current layout, + # so it resolves to the floor. + self.peer_schema = peer_schema or _no_peer_schema self.codecs = [cls(config) for cls in CODECS] @property @@ -167,7 +168,7 @@ def negotiated_protocol_version_for_peer( known schema raises by default; with ``raise_on_unknown=False`` it is assumed to run the current protocol. """ - schema = self.peer_schemas.get(peer_email) + schema = self.peer_schema(peer_email) if schema is not None: return self.registry.infer_protocol_version_to_use( peer_version=schema.version, @@ -208,7 +209,7 @@ def target_protocol_versions_for_peers( return set() versions: set[str] = set() for email in peer_emails: - schema = self.peer_schemas.get(email) + schema = self.peer_schema(email) if schema is not None: versions.add(min(DATASET_PROTOCOL_VERSION, schema.version, key=int)) else: diff --git a/packages/syft-datasets/tests/migrations/p2p/test_current_protocol.py b/packages/syft-datasets/tests/migrations/p2p/test_current_protocol.py index 793467ba30c..ee7bb2c7370 100644 --- a/packages/syft-datasets/tests/migrations/p2p/test_current_protocol.py +++ b/packages/syft-datasets/tests/migrations/p2p/test_current_protocol.py @@ -35,7 +35,7 @@ def _dataset_manager(tmp_path: Path, peer_schemas=None) -> SyftDatasetManager: syftbox.mkdir() mgr = SyftDatasetManager(syftbox_folder_path=syftbox, email=DO_EMAIL) if peer_schemas is not None: - mgr.storage.peer_schemas = peer_schemas + mgr.storage.peer_schema = peer_schemas.get return mgr diff --git a/packages/syft-datasets/tests/migrations/p2p/test_protocol_sanity_checks.py b/packages/syft-datasets/tests/migrations/p2p/test_protocol_sanity_checks.py index 2cfbf50bab7..6c25fe77c97 100644 --- a/packages/syft-datasets/tests/migrations/p2p/test_protocol_sanity_checks.py +++ b/packages/syft-datasets/tests/migrations/p2p/test_protocol_sanity_checks.py @@ -17,7 +17,7 @@ def _storage(tmp_path: Path, peer_schemas=None) -> DatasetStorage: syftbox = tmp_path / "SyftBox" syftbox.mkdir() config = SyftBoxConfig(syftbox_folder=syftbox, email=DO_EMAIL) - return DatasetStorage(config=config, peer_schemas=peer_schemas) + return DatasetStorage(config=config, peer_schema=(peer_schemas or {}).get) def test_reserved_dataset_name_rejected(tmp_path: Path): diff --git a/packages/syft-enclave/README.md b/packages/syft-enclave/README.md index 45f0c1cc568..354c7d7a0e0 100644 --- a/packages/syft-enclave/README.md +++ b/packages/syft-enclave/README.md @@ -69,7 +69,7 @@ just start-debug EMAIL my-vm n2d-standard-4 # override name / machine type just stop [name] # Teardown: Deletes the VM. ``` -Debug enclaves run with encryption off — data owner clients must match: `login_do(encryption=False)`. +An enclave always requires encryption, debug enclaves too. Clients select encryption from the enclave policy, so they need no setting. ## GPU deployments diff --git a/packages/syft-enclave/docs/dev.md b/packages/syft-enclave/docs/dev.md index 467ec261be5..ec803217d5a 100644 --- a/packages/syft-enclave/docs/dev.md +++ b/packages/syft-enclave/docs/dev.md @@ -25,7 +25,7 @@ just start-debug # cpu (default) just hardware=gpu start-debug # gpu (a3-highgpu-1g, 1x H100) — see docs/terraform_cs.md (Confidential Spaces) "GPU deployments" ``` -Debug enclaves run with encryption off — data owner clients must match: `login_do(encryption=False)`. +An enclave always requires encryption, debug enclaves too. Clients select encryption from the enclave policy, so they need no setting. ## Building & pushing a new Docker Image diff --git a/packages/syft-enclave/docs/terraform_cs.md b/packages/syft-enclave/docs/terraform_cs.md index 503954a287b..0a97d2431b8 100644 --- a/packages/syft-enclave/docs/terraform_cs.md +++ b/packages/syft-enclave/docs/terraform_cs.md @@ -101,7 +101,7 @@ Deletes the VM, secret, service account, and IAM bindings. The enabled APIs are Debug image — SSH enabled, container logs redirected to serial output, encryption off (override with `use_encryption = true` in tfvars), restart policy `Never` (a crashed container stays down for post-mortem; `gcloud compute instances reset` to relaunch). -> Clients must match the enclave's encryption setting: against a debug enclave, data owners log in with `login_do(encryption=False)`. +> Clients select encryption from the policy the enclave advertises, so they need no encryption setting. ```bash just tf-apply-dev diff --git a/packages/syft-enclave/docs/tinfoil_troubleshooting.md b/packages/syft-enclave/docs/tinfoil_troubleshooting.md index 78a912ab9e4..148bf6271f7 100644 --- a/packages/syft-enclave/docs/tinfoil_troubleshooting.md +++ b/packages/syft-enclave/docs/tinfoil_troubleshooting.md @@ -10,22 +10,22 @@ for what the attestation does and does not prove, see ## Symptoms -| Symptom | Cause and fix | -| -------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `Firewall setup failed: creating docker network "default": operation is not permitted on predefined default network` | a network named `default` collides with Docker's predefined one. The canonical schema only reserves `shim-net`, so this **validates fine and fails on the host**. Name it anything else. | -| `FileNotFoundError: No usable temporary directory found in ['/tmp', ...]` and a crash-loop | containers run with a read-only rootfs, and `portalocker` (a syft dependency) calls `tempfile.gettempdir()` at import. Set `read_only: false`. | -| `2 validation errors for EnclaveSettings: email / data_owners Field required` — despite passing `--variable` | the shim only injects variables whose **key the measured config declares**. An undeclared `--variable` is silently dropped. Declare it as a bare name under `env:`. | -| The enclave hangs at "Building SyftEnclaveClient" | OAuth token refresh cannot reach `oauth2.googleapis.com`. See the egress note in `tinfoil-config.yml`: `allowlist` resolves hostnames to IPs once at boot and Google rotates them, so use `egress: open`. | -| `curl: (60) SSL certificate problem: self signed certificate` | expected. The enclave's TLS key is generated inside it and the report commits to that key, so there is no CA. Use `-k` (as `just tinfoil-attest` does) and get your trust from `just tinfoil-verify`. | -| `The server had an error while processing your request.` from the domain | the shim is up but your container is not serving on `upstream-port`. It has probably crashed — `just tinfoil-debug ` then `just tinfoil-logs`. | -| `ValueError: Serialization error: sender fingerprint mismatch` on a data owner's login | with `encryption=True` the keypair is persisted at `//private/crypto_keys.json`. A fresh `SYFTBOX_FOLDER` mints a new identity that cannot verify that account's own history on Drive. Point at the account's existing folder, or wipe its Drive state first. | -| `upstream port is not set` | the config's `shim.upstream-port` is missing; it is required | -| Config rejected: image must be a digest | `image:` uses a tag; it must be `repo@sha256:...` | -| A path 404s | it is not in `shim.paths`. `/.well-known/tinfoil-attestation` is exempt | -| `Tag vX.Y.Z already exists` | releases are permanent; pick the next version | -| `image_digest` check fails with a 400 from `github-proxy.tinfoil.sh` | expected — the proxy only serves `tinfoil.hash`; the verifier falls through to github.com | -| `MissingOptionalDependency: ... tinfoil` | `uv pip install "syft-enclave[tinfoil]"` | -| `measurement_match` fails | the running enclave is not the release you are verifying against — check the deployed tag with `just tinfoil-status` | -| Peer skipped with "published no attestation evidence" | the enclave booted outside a TEE, or with `attestation_provider=none` | +| Symptom | Cause and fix | +| -------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Firewall setup failed: creating docker network "default": operation is not permitted on predefined default network` | a network named `default` collides with Docker's predefined one. The canonical schema only reserves `shim-net`, so this **validates fine and fails on the host**. Name it anything else. | +| `FileNotFoundError: No usable temporary directory found in ['/tmp', ...]` and a crash-loop | containers run with a read-only rootfs, and `portalocker` (a syft dependency) calls `tempfile.gettempdir()` at import. Set `read_only: false`. | +| `2 validation errors for EnclaveSettings: email / data_owners Field required` — despite passing `--variable` | the shim only injects variables whose **key the measured config declares**. An undeclared `--variable` is silently dropped. Declare it as a bare name under `env:`. | +| The enclave hangs at "Building SyftEnclaveClient" | OAuth token refresh cannot reach `oauth2.googleapis.com`. See the egress note in `tinfoil-config.yml`: `allowlist` resolves hostnames to IPs once at boot and Google rotates them, so use `egress: open`. | +| `curl: (60) SSL certificate problem: self signed certificate` | expected. The enclave's TLS key is generated inside it and the report commits to that key, so there is no CA. Use `-k` (as `just tinfoil-attest` does) and get your trust from `just tinfoil-verify`. | +| `The server had an error while processing your request.` from the domain | the shim is up but your container is not serving on `upstream-port`. It has probably crashed — `just tinfoil-debug ` then `just tinfoil-logs`. | +| `ValueError: Serialization error: sender fingerprint mismatch` on a data owner's login | the first login persists the keypair at `//private/crypto_keys.json`. A fresh `SYFTBOX_FOLDER` mints a new identity that cannot verify that account's own history on Drive. Point at the account's existing folder, or wipe its Drive state first. | +| `upstream port is not set` | the config's `shim.upstream-port` is missing; it is required | +| Config rejected: image must be a digest | `image:` uses a tag; it must be `repo@sha256:...` | +| A path 404s | it is not in `shim.paths`. `/.well-known/tinfoil-attestation` is exempt | +| `Tag vX.Y.Z already exists` | releases are permanent; pick the next version | +| `image_digest` check fails with a 400 from `github-proxy.tinfoil.sh` | expected — the proxy only serves `tinfoil.hash`; the verifier falls through to github.com | +| `MissingOptionalDependency: ... tinfoil` | `uv pip install "syft-enclave[tinfoil]"` | +| `measurement_match` fails | the running enclave is not the release you are verifying against — check the deployed tag with `just tinfoil-status` | +| Peer skipped with "published no attestation evidence" | the enclave booted outside a TEE, or with `attestation_provider=none` | Redeploying? Delete the accounts' syftboxes **first**, then redeploy — the enclave caches peer Drive folders at boot, so wiping state afterwards breaks the flow. diff --git a/packages/syft-enclave/scripts/tinfoil_e2e_check.py b/packages/syft-enclave/scripts/tinfoil_e2e_check.py index 7d7da637724..987ef4bb495 100644 --- a/packages/syft-enclave/scripts/tinfoil_e2e_check.py +++ b/packages/syft-enclave/scripts/tinfoil_e2e_check.py @@ -97,7 +97,7 @@ def main() -> int: from syft_enclaves.attestation.tinfoil import TinfoilAppraisalPolicy print("=== 1. login as the data owner ===", flush=True) - client = login_do(email=args.do_email, token_path=args.token, encryption=True) + client = login_do(email=args.do_email, token_path=args.token) print("=== 2. peer with the enclave ===", flush=True) client.add_peer(args.enclave_email) diff --git a/packages/syft-enclave/src/syft_enclaves/__main__.py b/packages/syft-enclave/src/syft_enclaves/__main__.py index c0efc79631e..66b8b57df14 100644 --- a/packages/syft-enclave/src/syft_enclaves/__main__.py +++ b/packages/syft-enclave/src/syft_enclaves/__main__.py @@ -46,7 +46,7 @@ def main() -> None: f"Enclave settings — email={settings.email} data_owners={settings.data_owners} " f"token_path={settings.token_path} poll_interval={settings.poll_interval}s " f"require_tee={settings.require_tee} fresh_state={settings.fresh_state} " - f"use_encryption={settings.use_encryption} receipts={settings.receipts} " + f"receipts={settings.receipts} " f"attestation_provider={settings.attestation_provider} " f"tinfoil_repo={settings.tinfoil_repo} " f"tinfoil_release_tag={settings.tinfoil_release_tag} " @@ -58,7 +58,6 @@ def main() -> None: email=settings.email, token_path=settings.token_path, data_owners=settings.data_owners, - encryption=settings.use_encryption, persist_owner_state=settings.persist_owner_state, ) if settings.receipts: diff --git a/packages/syft-enclave/src/syft_enclaves/client.py b/packages/syft-enclave/src/syft_enclaves/client.py index e47e0f69d4b..8d7613a7fa0 100644 --- a/packages/syft-enclave/src/syft_enclaves/client.py +++ b/packages/syft-enclave/src/syft_enclaves/client.py @@ -15,6 +15,7 @@ from syft_rds import SyftRDSClient, SyftRDSClientConfig from syft_rds.waiting import JobCheck, JobStatusArg, wait_for_job +from syft.sync.encryption_policy import EncryptionPolicy from syft.sync.peers.peer import Peer from syft.sync.peers.peer_list import PeerList from syft.sync.utils.waiting import ( @@ -271,10 +272,10 @@ def _peer_evidence( Prints why there is none, unless ``quiet`` is set. """ - version_info = self._rds.peer_manager.connection_router.read_peer_version_file( + peer_info = self._rds.peer_manager.connection_router.read_peer_info_file( peer_email ) - if version_info is None: + if peer_info is None: if not quiet: print( f"ℹ️ No version file available for peer {peer_email!r}; skipping attestation." @@ -283,7 +284,7 @@ def _peer_evidence( # A malformed envelope raises rather than skipping: a peer that # published something unparseable is not the same as one that # published nothing. - evidence = AttestationEvidence.read_from(version_info) + evidence = AttestationEvidence.read_from(peer_info.version_info) if evidence is None and not quiet: print( f"ℹ️ Peer {peer_email!r} published no attestation evidence " @@ -789,7 +790,7 @@ def from_config( """Build a SyftEnclaveClient from an RDS config with a wrapped job_client. Encryption rides along on ``config.sync`` via - ``peer_manager_config.use_encryption``; ``SyftboxManager.from_config`` + ``peer_manager_config.encryption_policy``; ``SyftboxManager.from_config`` (invoked inside ``SyftRDSClient.from_config``) loads/generates this datasite's keys from its per-datasite key file, so no extra step is needed. """ @@ -810,15 +811,16 @@ def for_enclave( email: str, token_path: Path | str | None = None, data_owners: list[str] | None = None, - encryption: bool = False, persist_owner_state: bool = False, ) -> "SyftEnclaveClient": """Build an enclave client backed by a real Google Drive connection. + + An enclave always requires encryption with every peer. + Args: email: The enclave datasite's email address. token_path: Path to a pre-authorized Google Drive OAuth token. data_owners: Emails whose approval gates every job on this enclave. - encryption: Enable end-to-end drive encryption. persist_owner_state: Keep the owner-only state used to restore this datasite later (event log, rolling state, checkpoints). Off by default - an enclave gets ephemeral keys and wipes its @@ -829,7 +831,7 @@ def for_enclave( has_ds_role=True, has_do_role=True, token_path=Path(token_path) if token_path is not None else None, - encryption=encryption, + encryption=True, persist_owner_state=persist_owner_state, ) @@ -863,18 +865,23 @@ def quad_with_mock_drive_service_connection( - DS: peers with DO1, DO2, Enclave Args: - encryption: Enable end-to-end drive encryption on all four clients. + encryption: Policy of the DOs and the DS. The enclave always + requires encryption. enclave_persist_owner_state: Whether the enclave keeps restorable owner state. Off by default, as in production. Returns: Tuple of (enclave, do1, do2, ds) """ + policies = [EncryptionPolicy.REQUIRED] + [ + EncryptionPolicy.from_bool(encryption) + ] * 3 configs = create_configs( enclave_email, do1_email, do2_email, ds_email, use_in_memory_cache, + policies, enclave_persist_owner_state, ) clients = create_clients(configs) @@ -889,12 +896,6 @@ def quad_with_mock_drive_service_connection( setup_connections(managers) setup_callbacks(managers) write_versions(managers) - # Initialize encryption AFTER connections/versions but BEFORE peering, - # so the bundle exchange in wire_peers() picks up each client's keys - # (mirrors SyftboxManager.pair_with_mock_drive_service_connection). - if encryption: - for manager in managers: - manager._init_encrypted_peer_store() wire_peers(managers) return clients diff --git a/packages/syft-enclave/src/syft_enclaves/login.py b/packages/syft-enclave/src/syft_enclaves/login.py index 0a51738f022..7fc630662ae 100644 --- a/packages/syft-enclave/src/syft_enclaves/login.py +++ b/packages/syft-enclave/src/syft_enclaves/login.py @@ -3,7 +3,11 @@ from pathlib import Path from syft.sync.environments.environment import Environment -from syft.sync.login import _init_client_login, _resolve_login_params +from syft.sync.login import ( + _import_key_file, + _init_client_login, + _resolve_login_params, +) from syft.sync.login_utils import handle_potential_version_mismatches_on_login from syft.sync.utils.syftbox_utils import check_env from syft_rds import SyftRDSClientConfig @@ -20,8 +24,8 @@ def _login( skip_peer_on_patch_version_diff: bool | None, has_do_role: bool, has_ds_role: bool, - encryption: bool = False, - crypto_keys_path: str | Path | None = None, + encryption: bool | None = None, + key_file: str | Path | None = None, ) -> SyftEnclaveClient: """Shared login for enclave-flow participants. @@ -29,9 +33,11 @@ def _login( run the login-time version-mismatch check, then build the enclave client for Colab or Jupyter — always wrapping the job_client with EnclaveJobClient. - When ``encryption`` is set, keys are persisted per-datasite to + Keys are persisted per-datasite to ``//private/crypto_keys.json`` (a stable identity - across sessions; never synced to Drive). + across sessions; never synced to Drive). The client makes them at the + first login. ``key_file`` restores the keys of an earlier session: it is + copied to that path before the client is built. """ env = check_env() email, token_path = _resolve_login_params(email, token_path) @@ -43,7 +49,6 @@ def _login( has_do_role=has_do_role, has_ds_role=has_ds_role, encryption=encryption, - crypto_keys_path=crypto_keys_path, skip_peer_on_patch_version_diff=skip_peer_on_patch_version_diff, ) else: @@ -53,12 +58,10 @@ def _login( has_ds_role=has_ds_role, token_path=Path(token_path) if token_path is not None else None, encryption=encryption, - crypto_keys_path=crypto_keys_path, skip_peer_on_patch_version_diff=skip_peer_on_patch_version_diff, ) - # Encryption keys (when enabled) are resolved inside from_config via the - # config's peer_manager_config. + _import_key_file(key_file, config.sync.syftbox_folder, email) client = SyftEnclaveClient.from_config(config) # Reuses syft's login init: verifies the token authenticates as @@ -79,12 +82,18 @@ def login_do( sync: bool = True, load_peers: bool = True, skip_peer_on_patch_version_diff: bool | None = None, - encryption: bool = False, - crypto_keys_path: str | Path | None = None, + encryption: bool | None = None, + key_file: str | Path | None = None, ) -> SyftEnclaveClient: """Log in a data owner for an enclave computation. - Set ``encryption=True`` to encrypt all drive communication. + Set ``encryption=True`` to require encryption with every peer, and + ``False`` to encrypt only with each peer that requires it, such as an + enclave. Without it, the client takes the policy of its SyftBox, or + ``False`` on a new one. + + ``key_file`` is the ``crypto_keys.json`` of an earlier session, e.g. an + uploaded file on Colab. Without it, the first login makes new keys. """ return _login( email=email, @@ -95,7 +104,7 @@ def login_do( has_do_role=True, has_ds_role=True, encryption=encryption, - crypto_keys_path=crypto_keys_path, + key_file=key_file, ) @@ -105,12 +114,18 @@ def login_ds( sync: bool = True, load_peers: bool = True, skip_peer_on_patch_version_diff: bool | None = None, - encryption: bool = False, - crypto_keys_path: str | Path | None = None, + encryption: bool | None = None, + key_file: str | Path | None = None, ) -> SyftEnclaveClient: """Log in a data scientist for an enclave computation. - Set ``encryption=True`` to encrypt all drive communication. + Set ``encryption=True`` to require encryption with every peer, and + ``False`` to encrypt only with each peer that requires it, such as an + enclave. Without it, the client takes the policy of its SyftBox, or + ``False`` on a new one. + + ``key_file`` is the ``crypto_keys.json`` of an earlier session, e.g. an + uploaded file on Colab. Without it, the first login makes new keys. """ return _login( email=email, @@ -121,5 +136,5 @@ def login_ds( has_do_role=False, has_ds_role=True, encryption=encryption, - crypto_keys_path=crypto_keys_path, + key_file=key_file, ) diff --git a/packages/syft-enclave/src/syft_enclaves/runner.py b/packages/syft-enclave/src/syft_enclaves/runner.py index 8281e83408c..87f0df93020 100644 --- a/packages/syft-enclave/src/syft_enclaves/runner.py +++ b/packages/syft-enclave/src/syft_enclaves/runner.py @@ -14,6 +14,7 @@ import logging import signal import time +from pathlib import Path from typing import Callable, Optional from syft.sync.peers.peer_store import datasite_crypto_keys_path @@ -127,6 +128,9 @@ def _on_initializing(self) -> None: "(local folder + Google Drive files) before init" ) self.client.delete_syftbox() + # The wipe took the key file with it. The attestation server reads + # it to sign nonces, so write the keys held in memory back. + self.client._rds.peer_manager.peer_store.save_keys(self._keys_path()) logger.info("State wipe complete — enclave starts with a clean slate") def _on_attesting(self) -> None: @@ -158,7 +162,7 @@ def _publish_attestation( evidence = provider.collect(**binding) peer_manager = self.client._rds.peer_manager evidence.publish_to(peer_manager.get_own_version()) - peer_manager.write_own_version() + peer_manager.write_own_peer_info() logger.info( "Attestation evidence (%s) published to SYFT_version.json", evidence.kind.value, @@ -175,11 +179,7 @@ def _build_claims(self) -> Optional[dict]: report already binds and serves it over the pinned connection. """ peer_store = self.client._rds.peer_manager.peer_store - bundle = ( - peer_store.get_public_bundle() - if peer_store.use_encryption and peer_store.has_my_keys() - else None - ) + bundle = peer_store.get_public_bundle() claims = build_claims( email=self.client.email, data_owners=self.client.data_owners, @@ -198,33 +198,26 @@ def _publish_key_bundle(self, claims: Optional[dict] = None) -> None: """Expose our public key bundle on the attestation endpoint. A peer fetching the report over a pinned connection gets the bundle - from the same channel, which binds it to the hardware report. Skipped - when encryption is off, since then there is no bundle. + from the same channel, which binds it to the hardware report. """ peer_store = self.client._rds.peer_manager.peer_store - if not peer_store.use_encryption or not peer_store.has_my_keys(): - logger.info("Encryption disabled — no key bundle to publish") - return - peer_manager = self.client._rds.peer_manager - # The per-datasite default location; the enclave never overrides - # crypto_keys_path, and PeerManager does not carry the resolved value. - keys_path = datasite_crypto_keys_path( - peer_manager.syftbox_folder, peer_store.email - ) try: - # fresh_state wiped the syftbox folder a moment ago, taking the key - # file with it — the keys live on in memory, so persist them again - # before advertising where they are. Without this the endpoint - # serves a bundle it cannot sign a nonce with. - peer_store.save_keys(keys_path) write_public_bundle( - peer_store.get_public_bundle(), keys_path=keys_path, claims=claims + peer_store.get_public_bundle(), + keys_path=self._keys_path(), + claims=claims, ) except OSError as e: # Not fatal: peers can still fall back to the Drive-published # bundle, they just lose the attestation binding. logger.warning("Could not publish the public key bundle: %s", e) + def _keys_path(self) -> Path: + peer_manager = self.client._rds.peer_manager + return datasite_crypto_keys_path( + peer_manager.syftbox_folder, peer_manager.peer_store.email + ) + def _on_peering(self) -> None: """Load peers and accept pending peer requests.""" self.client.load_peers() diff --git a/packages/syft-enclave/src/syft_enclaves/settings.py b/packages/syft-enclave/src/syft_enclaves/settings.py index 6f772190108..e938617e291 100644 --- a/packages/syft-enclave/src/syft_enclaves/settings.py +++ b/packages/syft-enclave/src/syft_enclaves/settings.py @@ -150,13 +150,6 @@ def _split_data_owners(cls, v: object) -> object: "owner can see from the release whether receipts are on." ), ) - use_encryption: bool = Field( - default=True, - description=( - "End-to-end drive encryption for all enclave peer communication. " - "Enabled by default; set false to disable." - ), - ) persist_owner_state: bool = Field( default=False, description=( diff --git a/packages/syft-enclave/src/syft_enclaves/utils.py b/packages/syft-enclave/src/syft_enclaves/utils.py index 5412000c5c2..a63c1b97aca 100644 --- a/packages/syft-enclave/src/syft_enclaves/utils.py +++ b/packages/syft-enclave/src/syft_enclaves/utils.py @@ -12,31 +12,38 @@ def create_configs( do2_email, ds_email, use_in_memory_cache, + policies, enclave_persist_owner_state=False, ) -> tuple: + """Configs of the enclave, DO1, DO2 and DS, with ``policies`` in that order.""" + enclave_policy, do1_policy, do2_policy, ds_policy = policies enclave_config = SyftRDSClientConfig._base_config_for_testing( email=enclave_email, has_do_role=True, has_ds_role=True, use_in_memory_cache=use_in_memory_cache, persist_owner_state=enclave_persist_owner_state, + encryption_policy=enclave_policy, ) do1_config = SyftRDSClientConfig._base_config_for_testing( email=do1_email, has_do_role=True, has_ds_role=True, use_in_memory_cache=use_in_memory_cache, + encryption_policy=do1_policy, ) do2_config = SyftRDSClientConfig._base_config_for_testing( email=do2_email, has_do_role=True, has_ds_role=True, use_in_memory_cache=use_in_memory_cache, + encryption_policy=do2_policy, ) ds_config = SyftRDSClientConfig._base_config_for_testing( email=ds_email, has_ds_role=True, use_in_memory_cache=use_in_memory_cache, + encryption_policy=ds_policy, ) return enclave_config, do1_config, do2_config, ds_config @@ -76,7 +83,7 @@ def setup_callbacks(managers: tuple): def write_versions(managers: tuple): for manager in managers: - manager.peer_manager.write_own_version() + manager.peer_manager.write_own_peer_info() def wire_peers(managers: tuple): diff --git a/packages/syft-enclave/tests/test_attestation_dispatch.py b/packages/syft-enclave/tests/test_attestation_dispatch.py index 24165e8b3a2..eacd4f36b4e 100644 --- a/packages/syft-enclave/tests/test_attestation_dispatch.py +++ b/packages/syft-enclave/tests/test_attestation_dispatch.py @@ -84,22 +84,22 @@ def test_no_policy_is_allowed(self): class TestAttestPeer: - def _client(self, version_info): + def _client(self, peer_info): client = SyftEnclaveClient(rds=MagicMock()) router = client._rds.peer_manager.connection_router - router.read_peer_version_file.return_value = version_info + router.read_peer_info_file.return_value = peer_info return client def _peer_publishing(self, evidence_field): """A peer whose version file carries this evidence in its extra bag.""" - return MagicMock(extra={"attestation": evidence_field}) + return MagicMock(version_info=MagicMock(extra={"attestation": evidence_field})) def test_no_version_file_skips(self, capsys): assert self._client(None).attest_peer("enclave@openmined.org") is None assert "No version file" in capsys.readouterr().out def test_no_evidence_skips(self, capsys): - client = self._client(MagicMock(extra={})) + client = self._client(self._peer_publishing(None)) assert client.attest_peer("enclave@openmined.org") is None assert "published no attestation evidence" in capsys.readouterr().out @@ -135,7 +135,7 @@ def test_expected_image_digest_builds_the_matching_policy(self): assert policy.expected_email == "enclave@openmined.org" def test_digest_and_policy_together_are_refused(self): - client = self._client(MagicMock(extra={})) + client = self._client(self._peer_publishing(None)) with pytest.raises(ValueError, match="not both"): client.attest_peer( "enclave@openmined.org", @@ -173,8 +173,10 @@ class TestAdoptingVerifiedKeys: def _client(self, bundle_on_drive=None): client = SyftEnclaveClient(rds=MagicMock()) router = client._rds.peer_manager.connection_router - router.read_peer_version_file.return_value = MagicMock( - extra={"attestation": TINFOIL_EVIDENCE.to_version_field()} + router.read_peer_info_file.return_value = MagicMock( + version_info=MagicMock( + extra={"attestation": TINFOIL_EVIDENCE.to_version_field()} + ) ) store = client._rds.peer_manager.peer_store store.has_peer_bundle.return_value = bundle_on_drive is not None diff --git a/packages/syft-enclave/tests/test_enclave_jobs.py b/packages/syft-enclave/tests/test_enclave_jobs.py index 0ade9b35625..25ddedd9e7d 100644 --- a/packages/syft-enclave/tests/test_enclave_jobs.py +++ b/packages/syft-enclave/tests/test_enclave_jobs.py @@ -73,12 +73,20 @@ def create_tmp_code_file(code: str): return str(code_path) -@pytest.mark.parametrize("encryption", [False, True]) -def test_enclave_job_distribution(encryption): +@pytest.mark.parametrize( + "encryption_kwargs", + [ + {"encryption": True}, + # Only the enclave requires encryption; the other clients comply. + {"encryption": False}, + ], + ids=["encrypted", "negotiated"], +) +def test_enclave_job_distribution(encryption_kwargs): """Test full flow: DS submits job to enclave, enclave distributes to DOs.""" enclave, do1, do2, ds = SyftEnclaveClient.quad_with_mock_drive_service_connection( use_in_memory_cache=False, - encryption=encryption, + **encryption_kwargs, ) # DO1 creates dataset1 diff --git a/packages/syft-enclave/tests/test_encryption.py b/packages/syft-enclave/tests/test_encryption.py index 226d78cf039..e72762da17c 100644 --- a/packages/syft-enclave/tests/test_encryption.py +++ b/packages/syft-enclave/tests/test_encryption.py @@ -7,6 +7,7 @@ os.environ["PRE_SYNC"] = "false" +from syft.sync.encryption_policy import EncryptionPolicy from syft_enclaves import SyftEnclaveClient @@ -36,8 +37,8 @@ def test_encrypted_quad_keys_and_bundles(): for client in (enclave, do1, do2, ds): store = client._rds.sync_engine._peer_store assert store is not None - assert store.use_encryption is True - assert store.has_my_keys() is True + assert store.requires_encryption is True + assert store.get_public_bundle() is not None # The enclave peers with all three; it should hold each of their bundles # after wiring, so it can encrypt to and verify from them. @@ -68,3 +69,24 @@ def test_encrypted_message_to_enclave_roundtrips(): enclave.sync() assert "simple_job" in [j.name for j in enclave.jobs] + + +def test_quad_enclave_requires_encryption_with_optional_peers(): + enclave, do1, *_ = SyftEnclaveClient.quad_with_mock_drive_service_connection() + enclave_store = enclave._rds.peer_manager.peer_store + assert enclave_store.encryption_policy == EncryptionPolicy.REQUIRED + assert ( + do1._rds.peer_manager.peer_store.encryption_policy == EncryptionPolicy.OPTIONAL + ) + + +def test_for_enclave_requires_encryption(monkeypatch): + configs = [] + monkeypatch.setattr( + SyftEnclaveClient, + "from_config", + classmethod(lambda cls, config, data_owners=None: configs.append(config)), + ) + SyftEnclaveClient.for_enclave(email="enclave@openmined.org") + policy = configs[0].sync.peer_manager_config.encryption_policy + assert policy == EncryptionPolicy.REQUIRED diff --git a/packages/syft-enclave/tests/test_login.py b/packages/syft-enclave/tests/test_login.py new file mode 100644 index 00000000000..7066928274f --- /dev/null +++ b/packages/syft-enclave/tests/test_login.py @@ -0,0 +1,36 @@ +from pathlib import Path +from unittest.mock import MagicMock + +import syft.sync.login +from syft.sync.environments.environment import Environment +from syft.sync.peers.peer_store import PeerStore +from syft_enclaves import login as enclave_login + +EMAIL = "do@openmined.org" + + +def test_login_restores_uploaded_key_file(tmp_path, monkeypatch): + uploaded = tmp_path / "upload" / "crypto_keys.json" + earlier = PeerStore.create(email=EMAIL, keys_path=uploaded) + + monkeypatch.setattr(Path, "home", lambda: tmp_path / "home") + for module in (enclave_login, syft.sync.login): + monkeypatch.setattr(module, "check_env", lambda: Environment.JUPYTER) + monkeypatch.setattr( + enclave_login, + "handle_potential_version_mismatches_on_login", + lambda email, token_path: False, + ) + monkeypatch.setattr(enclave_login, "_init_client_login", lambda *a, **kw: None) + configs = [] + monkeypatch.setattr( + enclave_login.SyftEnclaveClient, + "from_config", + classmethod(lambda cls, config: configs.append(config) or MagicMock()), + ) + + enclave_login.login_do(EMAIL, token_path=tmp_path / "token.json", key_file=uploaded) + + keys_path = configs[0].sync.peer_manager_config.crypto_keys_path + restored = PeerStore.create(email=EMAIL, keys_path=keys_path) + assert restored.get_public_bundle() == earlier.get_public_bundle() diff --git a/packages/syft-enclave/tests/test_runner.py b/packages/syft-enclave/tests/test_runner.py index 2586d0e1228..5177ad2c134 100644 --- a/packages/syft-enclave/tests/test_runner.py +++ b/packages/syft-enclave/tests/test_runner.py @@ -37,6 +37,16 @@ def test_fresh_state_true_invokes_delete_syftbox(): client.delete_syftbox.assert_called_once_with() +def test_fresh_state_writes_keys_back_after_wipe(): + """The attestation server signs nonces with the key file the wipe deleted.""" + client = _make_client() + runner = EnclaveRunner(client=client, fresh_state=True, attestation_provider="none") + runner.init() + + peer_store = client._rds.peer_manager.peer_store + peer_store.save_keys.assert_called_once_with(runner._keys_path()) + + def test_fresh_state_false_skips_delete_syftbox(): """Opting out (fresh_state=False) preserves state across init.""" client = _make_client() @@ -85,7 +95,7 @@ def test_no_tee_and_require_tee_names_every_probed_path(self): def test_no_tee_without_require_tee_publishes_nothing(self): client = _make_client() EnclaveRunner(client=client, attestation_provider="none").init() - client._rds.peer_manager.write_own_version.assert_not_called() + client._rds.peer_manager.write_own_peer_info.assert_not_called() def test_publishes_evidence_to_the_version_file(self, monkeypatch): evidence = tinfoil_evidence(TINFOIL_DOC, repo="OpenMined/x") @@ -103,7 +113,7 @@ def test_publishes_evidence_to_the_version_file(self, monkeypatch): EnclaveRunner(client=client, require_tee=True).init() assert AttestationEvidence.read_from(version) == evidence - client._rds.peer_manager.write_own_version.assert_called_once_with() + client._rds.peer_manager.write_own_peer_info.assert_called_once_with() def test_the_key_bundle_is_published_for_the_attestation_endpoint( self, monkeypatch @@ -131,8 +141,6 @@ def test_the_key_bundle_is_published_for_the_attestation_endpoint( client._rds.peer_manager.syftbox_folder = Path("/tmp/SyftBox_enclave") store = client._rds.peer_manager.peer_store store.email = "enclave@openmined.org" - store.use_encryption = True - store.has_my_keys.return_value = True store.get_public_bundle.return_value = {"identity": "enclave@openmined.org"} EnclaveRunner(client=client, require_tee=True).init() @@ -140,30 +148,6 @@ def test_the_key_bundle_is_published_for_the_attestation_endpoint( assert written["bundle"] == {"identity": "enclave@openmined.org"} # The endpoint needs the private key location to answer a nonce. assert written["keys_path"].name == "crypto_keys.json" - # And the keys must be back on disk: fresh_state wiped the folder - # moments earlier, so the file the endpoint signs with is gone. - store.save_keys.assert_called_once_with(written["keys_path"]) - - def test_no_key_bundle_is_published_without_encryption(self, monkeypatch): - provider = MagicMock() - provider.kind = AttestationKind.TINFOIL - provider.collect.return_value = tinfoil_evidence(TINFOIL_DOC) - monkeypatch.setattr( - "syft_enclaves.runner.select_provider", lambda name, settings: provider - ) - calls = [] - monkeypatch.setattr( - "syft_enclaves.runner.write_public_bundle", - lambda bundle, keys_path, claims=None: calls.append(bundle), - ) - - client = _make_client() - client._rds.peer_manager.syftbox_folder = Path("/tmp/SyftBox_enclave") - client._rds.peer_manager.peer_store.use_encryption = False - - EnclaveRunner(client=client, require_tee=True).init() - - assert calls == [] def test_provider_gets_the_settings_object(self, monkeypatch): seen = {} @@ -205,8 +189,6 @@ def _client(self): client.data_owners = ["b@openmined.org", "a@openmined.org"] store = client._rds.peer_manager.peer_store store.email = "enclave@openmined.org" - store.use_encryption = True - store.has_my_keys.return_value = True store.get_public_bundle.return_value = {"identity": "enclave@openmined.org"} return client @@ -224,8 +206,9 @@ def test_the_key_bundle_exists_before_the_token_is_minted(self, monkeypatch): provider = self._provider(monkeypatch, binds_claims=True) client = self._client() order = [] - client._rds.peer_manager.peer_store.save_keys.side_effect = ( - lambda p: order.append("keys") + monkeypatch.setattr( + "syft_enclaves.runner.write_public_bundle", + lambda bundle, keys_path, claims=None: order.append("keys"), ) provider.collect.side_effect = lambda **kw: ( order.append("mint"), diff --git a/packages/syft-enclave/tests/test_settings.py b/packages/syft-enclave/tests/test_settings.py index cc1a2292526..78f6cd2559f 100644 --- a/packages/syft-enclave/tests/test_settings.py +++ b/packages/syft-enclave/tests/test_settings.py @@ -38,7 +38,6 @@ def test_defaults_applied_when_required_fields_set(required_env): assert settings.require_tee is False assert settings.log_level == "INFO" assert settings.fresh_state is True # default: always start with a clean slate - assert settings.use_encryption is True # default: encryption on assert settings.persist_owner_state is False # default: an enclave never restores @@ -110,12 +109,6 @@ def test_fresh_state_can_be_disabled_via_env(required_env): assert settings.fresh_state is False -def test_use_encryption_can_be_disabled_via_env(required_env): - required_env.setenv("SYFT_ENCLAVE_USE_ENCRYPTION", "false") - settings = EnclaveSettings(_env_file=None) - assert settings.use_encryption is False - - def test_persist_owner_state_can_be_enabled_via_env(required_env): required_env.setenv("SYFT_ENCLAVE_PERSIST_OWNER_STATE", "true") settings = EnclaveSettings(_env_file=None) diff --git a/packages/syft-job/src/syft_job/client.py b/packages/syft-job/src/syft_job/client.py index b9c7bf4122a..47803d00afa 100644 --- a/packages/syft-job/src/syft_job/client.py +++ b/packages/syft-job/src/syft_job/client.py @@ -86,7 +86,7 @@ def __init__( self, config: SyftJobConfig, target_datasite_owner_email: Optional[str] = None, - peer_schemas: Optional[dict[str, ProtocolSchema]] = None, + peer_schema: Optional[Callable[[str], Optional[ProtocolSchema]]] = None, ): """Initialize JobClient with configuration and optional user email for job views.""" self.config = config @@ -104,8 +104,8 @@ def __init__( str, str ] = {} # do_email -> syft install source (local path, git URL, or "syft==X.Y.Z") advertised by that DO in their VersionInfo # All model reads/writes and path resolution go through the manager; - # peer_schemas (peer email -> job ProtocolSchema) is filled by syft. - self.manager = JobStorage(config=config, peer_schemas=peer_schemas) + # peer_schema (peer email -> job ProtocolSchema) comes from syft. + self.manager = JobStorage(config=config, peer_schema=peer_schema) # Validate that user_email exists in SyftBox root self._validate_user_email() @@ -114,9 +114,9 @@ def __init__( def from_config( cls, config: SyftJobConfig, - peer_schemas: Optional[dict[str, ProtocolSchema]] = None, + peer_schema: Optional[Callable[[str], Optional[ProtocolSchema]]] = None, ) -> "JobClient": - return cls(config, config.current_user_email, peer_schemas=peer_schemas) + return cls(config, config.current_user_email, peer_schema=peer_schema) def _validate_user_email(self) -> None: """Validate that the user_email directory exists in SyftBox root.""" diff --git a/packages/syft-job/src/syft_job/job_storage.py b/packages/syft-job/src/syft_job/job_storage.py index 4a69a627e54..edc9ca4c26c 100644 --- a/packages/syft-job/src/syft_job/job_storage.py +++ b/packages/syft-job/src/syft_job/job_storage.py @@ -1,6 +1,6 @@ import logging from pathlib import Path -from typing import Iterator, Optional +from typing import Callable, Iterator, Optional from syft_migration import ( MigratableObject, @@ -29,6 +29,11 @@ __all__ = ["JobRef", "JobStateNotFoundError", "JobStorage"] +def _no_peer_schema(peer_email: str) -> Optional[ProtocolSchema]: + """No peer schema is known: the default when syft passes no lookup.""" + return None + + class JobStorage: """Job filesystem IO and path resolution, delegating disk layout to codecs. @@ -43,20 +48,15 @@ def __init__( self, config: SyftJobConfig, registry: MigrationRegistry = job_registry, - peer_schemas: Optional[dict[str, ProtocolSchema]] = None, + peer_schema: Optional[Callable[[str], Optional[ProtocolSchema]]] = None, ) -> None: self.config = config self.registry = registry self.service = MigrationService(registry=registry) - # peer email -> job ProtocolSchema; syft passes PeerManager's - # live map here (updated in place as peer version files load). Peers - # without an entry are assumed to run the current protocol. - # `is not None`, not `or`: syft passes a live (initially empty) - # dict it mutates as peer version files load; `or {}` would drop the - # shared reference and freeze negotiation at construction-time state. - self.peer_schemas: dict[str, ProtocolSchema] = ( - peer_schemas if peer_schemas is not None else {} - ) + # peer email -> the job ProtocolSchema it advertises, or None when not + # known; syft looks it up in the peer store on each call. Peers without + # a schema are assumed to run the current protocol. + self.peer_schema = peer_schema or _no_peer_schema self.codecs = [cls(config) for cls in CODECS] @property @@ -88,7 +88,7 @@ def negotiated_protocol_version_for_peer( known schema raises by default; with ``raise_on_unknown=False`` it is assumed to run the current protocol. """ - schema = self.peer_schemas.get(peer_email) + schema = self.peer_schema(peer_email) if schema is not None: return self.registry.infer_protocol_version_to_use( peer_version=schema.version, @@ -117,7 +117,7 @@ def _get_write_target_schema( format; a known peer schema only refines the object versions when it speaks the same protocol version. """ - schema = self.peer_schemas.get(reader_email) + schema = self.peer_schema(reader_email) if schema is not None and schema.version == ref.protocol_version: return schema return self.registry.schema_for_protocol_version(ref.protocol_version) diff --git a/packages/syft-job/tests/migrations/p2p/test_older_protocol_compatibility.py b/packages/syft-job/tests/migrations/p2p/test_older_protocol_compatibility.py index 4e4847faa78..2fe63e746fd 100644 --- a/packages/syft-job/tests/migrations/p2p/test_older_protocol_compatibility.py +++ b/packages/syft-job/tests/migrations/p2p/test_older_protocol_compatibility.py @@ -182,7 +182,7 @@ def test_submit_to_peer_writes_that_protocol_layout(peer_protocol: str, tmp_path ) peer_schema = job_registry.schema_for_protocol_version(peer_protocol) ds_client_current_protocol = JobClient( - config=ds_config_current_protocol, peer_schemas={DO_EMAIL: peer_schema} + config=ds_config_current_protocol, peer_schema={DO_EMAIL: peer_schema}.get ) job_dir = ds_client_current_protocol.submit_bash_job( diff --git a/packages/syft-job/tests/migrations/p2p/test_protocol_sanity_checks.py b/packages/syft-job/tests/migrations/p2p/test_protocol_sanity_checks.py index c211f23f62e..3d10bfba858 100644 --- a/packages/syft-job/tests/migrations/p2p/test_protocol_sanity_checks.py +++ b/packages/syft-job/tests/migrations/p2p/test_protocol_sanity_checks.py @@ -53,7 +53,7 @@ def test_negotiated_protocol_version_for_peer_raises_on_unknown(tmp_path: Path): ds_config = SyftJobConfig(syftbox_folder=syftbox, current_user_email=DS_EMAIL) protocol0_schema = job_registry.schema_for_protocol_version("0") manager = JobClient( - config=ds_config, peer_schemas={DO_EMAIL: protocol0_schema} + config=ds_config, peer_schema={DO_EMAIL: protocol0_schema}.get ).manager # An older peer negotiates down to the version both sides can read. diff --git a/packages/syft-job/tests/test_requests.py b/packages/syft-job/tests/test_requests.py index 76ce7e2e502..8049df4b193 100644 --- a/packages/syft-job/tests/test_requests.py +++ b/packages/syft-job/tests/test_requests.py @@ -189,7 +189,7 @@ def test_client_refuses_request_to_do_without_api_calls(tmp_path: Path): syftbox = tmp_path / "SyftBox" config = SyftJobConfig(syftbox_folder=syftbox, current_user_email=DS_EMAIL) old_do = ProtocolSchema(protocol_name="syft-job", version="1") - ds = JobClient(config, peer_schemas={DO_EMAIL: old_do}) + ds = JobClient(config, peer_schema={DO_EMAIL: old_do}.get) with pytest.raises(ValueError, match="job protocol 1"): ds.submit_api_request(DO_EMAIL, "call.job", API_NAME, PARAMS) diff --git a/packages/syft-rds/src/syft_rds/__init__.py b/packages/syft-rds/src/syft_rds/__init__.py index 95b1dbeb213..5ec8b81915b 100644 --- a/packages/syft-rds/src/syft_rds/__init__.py +++ b/packages/syft-rds/src/syft_rds/__init__.py @@ -13,6 +13,8 @@ from syft.sync.utils.path_filters import is_normal_syncable_path from syft.sync.environments.environment import Environment from syft.sync.utils.syftbox_utils import check_env +from syft.sync.encryption_policy import EncryptionPolicy, EncryptionPolicyMismatchError +from syft.sync.peers.peer_store import datasite_settings_path, resolve_encryption_policy __all__ = [ "SyftRDSClient", @@ -25,6 +27,10 @@ "is_normal_syncable_path", "Environment", "check_env", + "EncryptionPolicy", + "EncryptionPolicyMismatchError", + "datasite_settings_path", + "resolve_encryption_policy", ] # Last, so the imports stay at the top. Nothing here logs at import time. diff --git a/packages/syft-rds/src/syft_rds/client.py b/packages/syft-rds/src/syft_rds/client.py index 51efd6a1aa6..f994babcc4f 100644 --- a/packages/syft-rds/src/syft_rds/client.py +++ b/packages/syft-rds/src/syft_rds/client.py @@ -6,11 +6,12 @@ import os import time from pathlib import Path -from typing import Any +from typing import Any, Callable, Optional from uuid import uuid4 import yaml from pydantic import BaseModel, ConfigDict +from syft_migration import ProtocolSchema from syft.sync.syftbox_manager import SyftboxManager from syft.sync.peers.peer import Peer @@ -109,6 +110,14 @@ def _print_skipped_jobs(skipped: list[tuple[str, str, str]]) -> None: print(" Pass ignore_peer_version=True to run them anyway.") +def _peer_schema_lookup( + manager: SyftboxManager, protocol_name: str +) -> Callable[[str], Optional[ProtocolSchema]]: + """Look up the ``protocol_name`` schema a peer advertises, on each call.""" + peer_store = manager.peer_manager.peer_store + return lambda email: peer_store.protocol_schema(email, protocol_name) + + class SyftRDSClient(BaseModel): # Holds live service objects (sync engine + RDS-owned managers), not # serializable data, so arbitrary types are allowed. @@ -141,21 +150,21 @@ def model_post_init(self, __context: Any) -> None: @classmethod def from_config(cls, config: "SyftRDSClientConfig") -> "SyftRDSClient": sync_engine = SyftboxManager.from_config(config.sync) - # The job client selects a job protocol version per peer from the live - # peer-schema map, as the dataset manager does for dataset layouts. + # The job client selects a job protocol version per peer from the + # schema the peer advertises, as the dataset manager does for layouts. job_client = JobClient.from_config( config.job, - peer_schemas=sync_engine.peer_manager.live_peer_schemas("syft-job"), + peer_schema=_peer_schema_lookup(sync_engine, "syft-job"), ) job_runner = ( SyftJobRunner.from_config(config.job) if config.sync.has_do_role else None ) - # The dataset manager gets the live peer-schema map, as the job client - # does. It selects a layout for each peer, and the transport carries one + # The dataset manager looks up peer schemas as the job client does. It + # selects a layout for each peer, and the transport carries one # collection for each layout. dataset_manager = SyftDatasetManager.from_config( config.dataset, - peer_schemas=sync_engine.peer_manager.live_peer_schemas("syft-dataset"), + peer_schema=_peer_schema_lookup(sync_engine, "syft-dataset"), ) return cls( sync_engine=sync_engine, @@ -184,14 +193,14 @@ def _build(mgr: SyftboxManager) -> "SyftRDSClient": sync_engine=mgr, job_client=JobClient.from_config( config.job, - peer_schemas=mgr.peer_manager.live_peer_schemas("syft-job"), + peer_schema=_peer_schema_lookup(mgr, "syft-job"), ), job_runner=( SyftJobRunner.from_config(config.job) if mgr.has_do_role else None ), dataset_manager=SyftDatasetManager.from_config( config.dataset, - peer_schemas=mgr.peer_manager.live_peer_schemas("syft-dataset"), + peer_schema=_peer_schema_lookup(mgr, "syft-dataset"), ), ) diff --git a/packages/syft-rds/src/syft_rds/login.py b/packages/syft-rds/src/syft_rds/login.py index 68285dbca6d..10355f50e77 100644 --- a/packages/syft-rds/src/syft_rds/login.py +++ b/packages/syft-rds/src/syft_rds/login.py @@ -14,7 +14,11 @@ from pathlib import Path from syft.sync.environments.environment import Environment -from syft.sync.login import _init_client_login, _resolve_login_params +from syft.sync.login import ( + _import_key_file, + _init_client_login, + _resolve_login_params, +) from syft.sync.login_utils import handle_potential_version_mismatches_on_login from syft.sync.utils.syftbox_utils import check_env @@ -31,6 +35,8 @@ def _login( skip_peer_on_patch_version_diff: bool | None, has_do_role: bool, has_ds_role: bool, + encryption: bool | None = None, + key_file: str | Path | None = None, ) -> SyftRDSClient: """Shared RDS login. @@ -47,6 +53,7 @@ def _login( has_do_role=has_do_role, has_ds_role=has_ds_role, skip_peer_on_patch_version_diff=skip_peer_on_patch_version_diff, + encryption=encryption, ) else: config = SyftRDSClientConfig.for_jupyter( @@ -55,8 +62,10 @@ def _login( has_ds_role=has_ds_role, token_path=Path(token_path) if token_path is not None else None, skip_peer_on_patch_version_diff=skip_peer_on_patch_version_diff, + encryption=encryption, ) + _import_key_file(key_file, config.sync.syftbox_folder, email) client = SyftRDSClient.from_config(config) _init_client_login( client.sync_engine, @@ -74,6 +83,8 @@ def login_do( sync: bool = True, load_peers: bool = True, skip_peer_on_patch_version_diff: bool | None = None, + encryption: bool | None = None, + key_file: str | Path | None = None, ) -> SyftRDSClient: """Log in as a Data Owner and return a self-contained RDS client.""" return _login( @@ -82,6 +93,8 @@ def login_do( sync=sync, load_peers=load_peers, skip_peer_on_patch_version_diff=skip_peer_on_patch_version_diff, + encryption=encryption, + key_file=key_file, has_do_role=True, has_ds_role=False, ) @@ -94,6 +107,8 @@ def login_ds( sync: bool = True, load_peers: bool = True, skip_peer_on_patch_version_diff: bool | None = None, + encryption: bool | None = None, + key_file: str | Path | None = None, ) -> SyftRDSClient: """Log in as a Data Scientist and return a self-contained RDS client.""" return _login( @@ -102,6 +117,8 @@ def login_ds( sync=sync, load_peers=load_peers, skip_peer_on_patch_version_diff=skip_peer_on_patch_version_diff, + encryption=encryption, + key_file=key_file, has_do_role=False, has_ds_role=True, ) diff --git a/packages/syft-rds/tests/test_login_key_file.py b/packages/syft-rds/tests/test_login_key_file.py new file mode 100644 index 00000000000..8f1df8c74a2 --- /dev/null +++ b/packages/syft-rds/tests/test_login_key_file.py @@ -0,0 +1,39 @@ +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +import syft.sync.login +from syft.sync.environments.environment import Environment +from syft.sync.peers.peer_store import PeerStore +from syft_rds import login as rds_login + +EMAIL = "do@openmined.org" + + +@pytest.mark.parametrize("login", [rds_login.login_do, rds_login.login_ds]) +def test_login_restores_uploaded_key_file(tmp_path, monkeypatch, login): + uploaded = tmp_path / "upload" / "crypto_keys.json" + earlier = PeerStore.create(email=EMAIL, keys_path=uploaded) + + monkeypatch.setattr(Path, "home", lambda: tmp_path / "home") + for module in (rds_login, syft.sync.login): + monkeypatch.setattr(module, "check_env", lambda: Environment.JUPYTER) + monkeypatch.setattr( + rds_login, + "handle_potential_version_mismatches_on_login", + lambda email, token_path: False, + ) + monkeypatch.setattr(rds_login, "_init_client_login", lambda *a, **kw: None) + configs = [] + monkeypatch.setattr( + rds_login.SyftRDSClient, + "from_config", + classmethod(lambda cls, config: configs.append(config) or MagicMock()), + ) + + login(EMAIL, token_path=tmp_path / "token.json", key_file=uploaded) + + keys_path = configs[0].sync.peer_manager_config.crypto_keys_path + restored = PeerStore.create(email=EMAIL, keys_path=keys_path) + assert restored.get_public_bundle() == earlier.get_public_bundle() diff --git a/packages/syft-rds/tests/test_version_mismatch_flow.py b/packages/syft-rds/tests/test_version_mismatch_flow.py index 20edd957cf6..5ebb44ae328 100644 --- a/packages/syft-rds/tests/test_version_mismatch_flow.py +++ b/packages/syft-rds/tests/test_version_mismatch_flow.py @@ -84,7 +84,7 @@ def _reinitialize_manager( ) if write_version: - manager.peer_manager.write_own_version() + manager.peer_manager.write_own_peer_info() return manager @@ -97,7 +97,7 @@ def _simulate_continue_on_mismatch(manager, backing_store): mock_conn = GDriveConnection.from_service(email, mock_service) def read_remote(e, t): - return mock_conn.read_own_version_file() + return mock_conn.read_own_peer_info_file().version_info with ( patch( @@ -328,7 +328,9 @@ def test_login_writes_the_remote_version_file_too(): check_versions=True, ) conn = do_manager.peer_manager.connection_router.connections[0] - assert conn.read_own_version_file().syft_client_version == SYFT_VERSION + assert ( + conn.read_own_peer_info_file().version_info.syft_client_version == SYFT_VERSION + ) with ( patch("syft.version.SYFT_VERSION", NEW_VERSION), @@ -337,11 +339,17 @@ def test_login_writes_the_remote_version_file_too(): upgraded = _upgraded_manager(do_manager) new_conn = upgraded.peer_manager.connection_router.connections[0] # Still the pre-upgrade version: nothing has refreshed it yet. - assert new_conn.read_own_version_file().syft_client_version == SYFT_VERSION + assert ( + new_conn.read_own_peer_info_file().version_info.syft_client_version + == SYFT_VERSION + ) _init_client_login(upgraded.sync_engine, sync=False, load_peers=False) - assert new_conn.read_own_version_file().syft_client_version == NEW_VERSION + assert ( + new_conn.read_own_peer_info_file().version_info.syft_client_version + == NEW_VERSION + ) local = read_local_version(upgraded.syftbox_folder) assert local is not None assert local.syft_client_version == NEW_VERSION @@ -383,7 +391,7 @@ def test_the_mismatch_prompt_does_not_return_after_a_login(): conn = do_manager.peer_manager.connection_router.connections[0] with patch( "syft.sync.login_utils._read_remote_version", - side_effect=lambda e, t: conn.read_own_version_file(), + side_effect=lambda e, t: conn.read_own_peer_info_file().version_info, ): # The check runs before the client exists, so it reads the files # the previous client version left behind. diff --git a/packages/syft-rds/tests/test_version_negotiation.py b/packages/syft-rds/tests/test_version_negotiation.py index f2cad3a2479..c942ffe0e35 100644 --- a/packages/syft-rds/tests/test_version_negotiation.py +++ b/packages/syft-rds/tests/test_version_negotiation.py @@ -5,11 +5,14 @@ import pytest from syft.sync.version.exceptions import VersionUnknownError +from syft.sync.encryption_policy import EncryptionPolicy +from syft.sync.version.peer_info import PeerInfo from syft.sync.version.peer_manager import PeerCompatibilityResult, PeerManager from syft.sync.version.version_info import CompatibilityStatus, VersionInfo from syft_job.config import protocol_dir_name from syft_job.migrations.registry import JOB_PROTOCOL_VERSION from syft_rds import SyftRDSClient +from syft.sync.peers.exceptions import SyftPeerNotReadyError def build_client_version(version: str) -> VersionInfo: @@ -25,7 +28,8 @@ def _set_peer_version(client, peer_email: str, version: VersionInfo): """Override the cached version for a peer without a Drive round-trip.""" peer = client.peer_manager.get_cached_peer(peer_email) assert peer is not None, f"peer {peer_email} not in store" - peer.version = version + policy = peer.encryption_policy or EncryptionPolicy.OPTIONAL + peer.info = PeerInfo(version_info=version, encryption_policy=policy) class TestForceSubmission: @@ -71,12 +75,15 @@ def test_job_submission_allowed_with_force(self): job_name="test.fail.job", ) - ds.submit_python_job( - user=do.email, - code_path=test_py_path, - job_name="test.force.job", - force_submission=True, - ) + # Force skips the version gate, but the DO policy is not known before it + # accepts, so the send raises. The job is written locally. + with pytest.raises(SyftPeerNotReadyError, match="not known"): + ds.submit_python_job( + user=do.email, + code_path=test_py_path, + job_name="test.force.job", + force_submission=True, + ) job_dir = ( ds.syftbox_folder diff --git a/scripts/generate_release_fixture.py b/scripts/generate_release_fixture.py index d501563a0ec..385fbabd73a 100644 --- a/scripts/generate_release_fixture.py +++ b/scripts/generate_release_fixture.py @@ -37,6 +37,8 @@ ProposedFileChange, ProposedFileChangesMessage, ) +from syft.sync.encryption_policy import EncryptionPolicy +from syft.sync.version.peer_info import PeerInfo from syft.sync.version.version_info import VersionInfo from syft.version import SYFT_VERSION @@ -48,12 +50,15 @@ ) -def build_version_info() -> VersionInfo: +def build_peer_info() -> PeerInfo: # Not VersionInfo.current(): the detected install source is an absolute # local path on dev machines, which must not leak into a committed fixture. - return VersionInfo.current().model_copy( + version_info = VersionInfo.current().model_copy( update={"syft_client_install_source": "pip"} ) + return PeerInfo( + version_info=version_info, encryption_policy=EncryptionPolicy.REQUIRED + ) def build_proposed_message() -> ProposedFileChangesMessage: @@ -105,7 +110,7 @@ def main() -> None: ) target.mkdir(parents=True) - (target / "SYFT_version.json").write_text(build_version_info().to_json()) + (target / "SYFT_version.json").write_text(build_peer_info().to_json()) proposed = build_proposed_message() (target / proposed.message_filename.as_string()).write_bytes( diff --git a/syft/__init__.py b/syft/__init__.py index 5a58db3189d..80c62b7065e 100644 --- a/syft/__init__.py +++ b/syft/__init__.py @@ -42,6 +42,7 @@ # Import the versioned model modules explicitly so registration is intentional, # not a side-effect of whatever login happened to pull in first. import syft.sync.version.version_info # noqa: F401, E402 +import syft.sync.version.peer_info # noqa: F401, E402 import syft.sync.messages.proposed_filechange # noqa: F401, E402 import syft.sync.events.file_change_event # noqa: F401, E402 diff --git a/syft/sync/connections/connection_router.py b/syft/sync/connections/connection_router.py index 35114df99fc..b3fee7d2b04 100644 --- a/syft/sync/connections/connection_router.py +++ b/syft/sync/connections/connection_router.py @@ -1,8 +1,9 @@ import logging -from typing import TYPE_CHECKING, Dict, List, Optional +import warnings +from typing import TYPE_CHECKING, List, Optional from pydantic import BaseModel, PrivateAttr -from syft_migration import MigratableObject, ProtocolSchema +from syft_migration import MigratableObject from syft.migrations import ( SYFT_CLIENT_PROTOCOL_VERSION, @@ -16,10 +17,14 @@ FileCollection, SyftboxPlatformConnection, ) +from syft.sync.encryption_policy import ( + MessageFormatError, +) from syft.sync.events.file_change_event import ( FileChangeEventsMessage, ) from syft.sync.messages.proposed_filechange import ProposedFileChangesMessage +from syft.sync.peers.exceptions import SyftPeerNotReadyError from syft.sync.peers.peer import Peer, PeerState from syft.sync.peers.peer_store import PeerStore from syft.sync.transport.naming import PEERS_META_KEY @@ -31,7 +36,7 @@ ) if TYPE_CHECKING: - from syft.sync.version.version_info import VersionInfo + from syft.sync.version.peer_info import PeerInfo logger = logging.getLogger(__name__) @@ -42,10 +47,6 @@ class ConnectionRouter(BaseModel): peer_store: PeerStore - # peer email -> syft ProtocolSchema; syft wires PeerManager's - # live map here (updated in place as peer version files load), so outgoing - # messages downgrade to what each peer reads. - _peer_schemas: Dict[str, ProtocolSchema] = PrivateAttr(default_factory=dict) # The transport jobs over connections[0], rebuilt if that connection is # replaced (tests swap in a mock connection with add_connection). _transport: Optional[Transport] = PrivateAttr(default=None) @@ -59,10 +60,6 @@ def transport(self) -> Transport: self._transport_source = connection return self._transport - def set_peer_schemas(self, peer_schemas: Dict[str, ProtocolSchema]) -> None: - """Adopt the live {peer email -> syft ProtocolSchema} map.""" - self._peer_schemas = peer_schemas - @classmethod def from_configs(cls, email: str, connection_configs: List[ConnectionConfig]): return cls( @@ -126,7 +123,7 @@ def _downgrade_for_peer( A peer without a known schema is assumed to run the current protocol, the same policy as jobs; the assumption is logged. """ - schema = self._peer_schemas.get(peer_email) + schema = self.peer_store.protocol_schema(peer_email, "syft") if schema is None: logger.warning( f"No syft protocol schema known for peer {peer_email!r}. " @@ -161,17 +158,34 @@ def watcher_send_proposed_file_changes_message( def owner_get_next_proposed_filechange_message( self, sender_email: str ) -> ProposedFileChangesMessage | None: + """The next message from ``sender_email``, or None when the inbox is empty. + + A message whose format does not match the pair is archived unread, + with a warning. + + Raises: + SyftPeerNotReadyError: when the message cannot be read yet. It + stays in the inbox. + """ connection = self.connection_for_receive_message() - result = connection.owner_download_next_raw_proposed_message_from_inbox( - sender_email - ) - if result is None: - return None - raw_data, file_id = result - raw_data = self.peer_store.decrypt_and_verify_if_needed(sender_email, raw_data) - msg = ProposedFileChangesMessage.from_compressed_data(raw_data) - msg.platform_id = file_id - return msg + while True: + result = connection.owner_download_next_raw_proposed_message_from_inbox( + sender_email + ) + if result is None: + return None + raw_data, file_id = result + try: + decrypted = self.peer_store.decrypt_and_verify_if_needed( + sender_email, raw_data + ) + except MessageFormatError as e: + warnings.warn(f"Archived message file {file_id} unread: {e}") + connection.owner_archive_inbox_file(file_id, sender_email) + continue + msg = ProposedFileChangesMessage.from_compressed_data(decrypted) + msg.platform_id = file_id + return msg def owner_write_event_messages_to_outbox( self, recipient_email: str, events_message: FileChangeEventsMessage @@ -185,16 +199,33 @@ def owner_write_event_messages_to_outbox( def watcher_get_events_messages( self, peer_email: str, since_timestamp: float | None ) -> List[FileChangeEventsMessage]: + """The outbox messages of ``peer_email``, without the skipped ones.""" connection = self.connection_for_datasite_watcher() raw_list = connection.watcher_download_raw_events_from_outbox( peer_email, since_timestamp ) - return [ - FileChangeEventsMessage.from_compressed_data( - self.peer_store.decrypt_and_verify_if_needed(peer_email, raw) - ) - for raw in raw_list + messages = [ + self.watcher_read_events_message(peer_email, raw) for raw in raw_list ] + return [m for m in messages if m is not None] + + def watcher_read_events_message( + self, peer_email: str, raw: bytes + ) -> FileChangeEventsMessage | None: + """Verify, decrypt and parse one outbox message of ``peer_email``. + + Returns None when the message is skipped, with a warning: its format + does not match the pair, or it cannot be read now. + """ + try: + data = self.peer_store.decrypt_and_verify_if_needed(peer_email, raw) + except ( + MessageFormatError, + SyftPeerNotReadyError, + ) as e: + warnings.warn(f"Skipped an outbox message of {peer_email}: {e}") + return None + return FileChangeEventsMessage.from_compressed_data(data) def watcher_get_outbox_file_metadatas( self, peer_email: str, since_timestamp: float | None @@ -393,31 +424,36 @@ def watcher_download_collection( } return files - def connection_for_version_read( + def connection_for_peer_file_read( self, create_new: bool = False ) -> SyftboxPlatformConnection: - """Get connection for reading version files. Can create new for thread safety.""" + """Connection for reading files a peer publishes: peer info and key bundles. + + Can create a new one for thread safety. + """ existing_connection = self.connections[0] if create_new: return self.copy_connection(existing_connection) else: return existing_connection - def write_version_file(self, version_info: "VersionInfo") -> None: - """Write version file to own SyftBox folder.""" - self.transport.peering.publish_version(version_info) + def write_peer_info_file(self, peer_info: "PeerInfo") -> None: + """Write the peer info file to own SyftBox folder.""" + self.transport.peering.publish_peer_info(peer_info) - def read_own_version_file(self) -> Optional["VersionInfo"]: - """Read version file from own SyftBox folder.""" - return self.transport.peering.read_own_version() + def read_own_peer_info_file(self) -> Optional["PeerInfo"]: + """Read the peer info file from own SyftBox folder.""" + return self.transport.peering.read_own_peer_info() - def read_peer_version_file(self, peer_email: str) -> Optional["VersionInfo"]: - """Read version file from a peer's SyftBox folder.""" - return self.transport.peering.read_peer_version(peer_email) + def read_peer_info_file(self, peer_email: str) -> Optional["PeerInfo"]: + """Read the peer info file from a peer's SyftBox folder.""" + return self.transport.peering.read_peer_info(peer_email) - def share_version_file_with_peer(self, peer_email: str) -> None: - """Share version file with a peer so they can read it.""" - self.transport.peering.grant_version_read(peer_email) + def share_peer_info_file_with_peer( + self, peer_email: str, peer_info: "PeerInfo" + ) -> None: + """Share the peer info file with a peer so they can read it.""" + self.transport.peering.grant_peer_info_read(peer_email, peer_info) def watcher_get_collection_file_metadatas( self, prefix: str, tag: str, content_hash: str, owner_email: str diff --git a/syft/sync/connections/drive/drive_connections.py b/syft/sync/connections/drive/drive_connections.py index d440e3814eb..8cd57a525a7 100644 --- a/syft/sync/connections/drive/drive_connections.py +++ b/syft/sync/connections/drive/drive_connections.py @@ -26,7 +26,7 @@ if TYPE_CHECKING: from syft.sync.connections.base_connection import FileCollection from syft.sync.connections.drive.gdrive_transport import GDriveConnection - from syft.sync.version.version_info import VersionInfo + from syft.sync.version.peer_info import PeerInfo class DriveP2PConnection(P2PConnection): @@ -241,17 +241,17 @@ def read_own_key_bundle(self, peer: str) -> str | None: def read_peer_key_bundle(self, peer: str) -> str | None: return self._conn.read_peer_encryption_bundle(peer) - def publish_version(self, version_info: VersionInfo) -> None: - self._conn.write_version_file(version_info) + def publish_peer_info(self, peer_info: PeerInfo) -> None: + self._conn.write_peer_info_file(peer_info) - def read_own_version(self) -> VersionInfo | None: - return self._conn.read_own_version_file() + def read_own_peer_info(self) -> PeerInfo | None: + return self._conn.read_own_peer_info_file() - def read_peer_version(self, peer: str) -> VersionInfo | None: - return self._conn.read_peer_version_file(peer) + def read_peer_info(self, peer: str) -> PeerInfo | None: + return self._conn.read_peer_info_file(peer) - def grant_version_read(self, peer: str) -> None: - self._conn.share_version_file_with_peer(peer) + def grant_peer_info_read(self, peer: str, peer_info: PeerInfo) -> None: + self._conn.share_peer_info_file_with_peer(peer, peer_info) def drive_transport(connection: GDriveConnection) -> Transport: diff --git a/syft/sync/connections/drive/gdrive_transport.py b/syft/sync/connections/drive/gdrive_transport.py index 42a54bab529..285f0357489 100644 --- a/syft/sync/connections/drive/gdrive_transport.py +++ b/syft/sync/connections/drive/gdrive_transport.py @@ -62,7 +62,7 @@ from syft.sync.connections.drive.grdrive_config import ( GdriveConnectionConfig, ) - from syft.sync.version.version_info import VersionInfo + from syft.sync.version.peer_info import PeerInfo # Timeout for Google API requests (in seconds) GOOGLE_API_TIMEOUT = 120 # 2 minutes @@ -855,14 +855,18 @@ def owner_remove_proposed_filechange_message_from_inbox( raise ValueError( f"Event {fname} not found in inbox, event should already be created for this type of connection" ) + self.owner_archive_inbox_file(gdrive_id, sender_email) + + def owner_archive_inbox_file(self, file_id: str, sender_email: str) -> None: + """Move the inbox file ``file_id`` from ``sender_email`` to its archive folder.""" file_info = execute_with_retries( - self.drive_service.files().get(fileId=gdrive_id, fields="parents") + self.drive_service.files().get(fileId=file_id, fields="parents") ) previous_parents = ",".join(file_info.get("parents", [])) archive_folder_id = self.owner_get_archive_folder_id(sender_email) execute_with_retries( self.drive_service.files().update( - fileId=gdrive_id, + fileId=file_id, addParents=archive_folder_id, removeParents=previous_parents, fields="id, parents", @@ -1835,8 +1839,8 @@ def _get_collection_folder_id( self.collection_folder_id_cache[folder_name] = folder_id return folder_id - def _get_version_file_id(self) -> Optional[str]: - """Find SYFT_version.json file in /SyftBox folder""" + def _get_own_peer_info_file_id(self) -> Optional[str]: + """Find the own peer info file, SYFT_version.json, in /SyftBox.""" syftbox_folder_id = self.get_syftbox_folder_id() query = f"name='{SYFT_VERSION_FILE}' and '{syftbox_folder_id}' in parents and trashed=false" results = execute_with_retries( @@ -1845,14 +1849,14 @@ def _get_version_file_id(self) -> Optional[str]: items = results.get("files", []) return items[0]["id"] if items else None - def write_version_file(self, version_info: "VersionInfo") -> None: - """Write version file to /SyftBox folder. Creates or updates the file.""" + def write_peer_info_file(self, peer_info: "PeerInfo") -> None: + """Write the peer info file to /SyftBox. Creates or updates the file.""" syftbox_folder_id = self.get_syftbox_folder_id() - file_id = self._get_version_file_id() + file_id = self._get_own_peer_info_file_id() # Convert to JSON string - json_data = version_info.to_json() + json_data = peer_info.to_json() file_payload, _ = self.create_file_payload(json_data) if file_id is None: @@ -1874,9 +1878,8 @@ def write_version_file(self, version_info: "VersionInfo") -> None: ) ) - def _get_peer_version_file_id(self, peer_email: str) -> Optional[str]: - """Find SYFT_version.json file in a peer's /SyftBox folder""" - # Find the peer's SyftBox folder + def _get_peer_info_file_id_of(self, peer_email: str) -> Optional[str]: + """Find the peer info file, SYFT_version.json, of ``peer_email``.""" query = ( f"name='{SYFT_VERSION_FILE}' and trashed=false and '{peer_email}' in owners" ) @@ -1886,53 +1889,57 @@ def _get_peer_version_file_id(self, peer_email: str) -> Optional[str]: items = results.get("files", []) return items[0]["id"] if items else None - def read_own_version_file(self) -> Optional["VersionInfo"]: - """Read version file from own /SyftBox folder.""" - from syft.sync.version.version_info import VersionInfo + def read_own_peer_info_file(self) -> Optional["PeerInfo"]: + """Read the own peer info file from /SyftBox.""" + from syft.sync.version.peer_info import PeerInfo - file_id = self._get_version_file_id() + file_id = self._get_own_peer_info_file_id() if file_id is None: return None try: file_data = self.download_file(file_id) except Exception as e: - print(f"Warning: could not download the own version file: {e}") + print(f"Warning: could not download the own peer info file: {e}") return None try: - return VersionInfo.from_json(file_data.decode("utf-8")) + return PeerInfo.from_json(file_data.decode("utf-8")) except (ValueError, MigrationError) as e: - print(f"Warning: could not read the own version file: {e}") + print(f"Warning: could not read the own peer info file: {e}") return None - def read_peer_version_file(self, peer_email: str) -> Optional["VersionInfo"]: - """Read version file from a peer's /SyftBox folder.""" - from syft.sync.version.version_info import VersionInfo + def read_peer_info_file(self, peer_email: str) -> Optional["PeerInfo"]: + """Read the peer info file from a peer's /SyftBox folder.""" + from syft.sync.version.peer_info import PeerInfo - file_id = self._get_peer_version_file_id(peer_email) + file_id = self._get_peer_info_file_id_of(peer_email) if file_id is None: return None try: file_data = self.download_file(file_id) except Exception as e: - print(f"Warning: could not download the version file of {peer_email}: {e}") + print( + f"Warning: could not download the peer info file of {peer_email}: {e}" + ) return None try: - return VersionInfo.from_json(file_data.decode("utf-8")) + return PeerInfo.from_json(file_data.decode("utf-8")) except (ValueError, MigrationError) as e: - print(f"Warning: could not read the version file of {peer_email}: {e}") + print(f"Warning: could not read the peer info file of {peer_email}: {e}") return None - def share_version_file_with_peer(self, peer_email: str) -> None: - """Share the version file with a peer so they can read it.""" - file_id = self._get_version_file_id() - if file_id is None: - # Version file doesn't exist yet, create it first - from syft.sync.version.version_info import VersionInfo + def share_peer_info_file_with_peer( + self, peer_email: str, peer_info: "PeerInfo" + ) -> None: + """Share the peer info file with a peer so they can read it. - self.write_version_file(VersionInfo.current()) - file_id = self._get_version_file_id() + A missing file is created from ``peer_info``, the info of this client. + """ + file_id = self._get_own_peer_info_file_id() + if file_id is None: + self.write_peer_info_file(peer_info) + file_id = self._get_own_peer_info_file_id() if file_id: self.add_permission(file_id, peer_email, write=False) @@ -2386,16 +2393,25 @@ def delete_rolling_state(self) -> None: # ENCRYPTION BUNDLE METHODS # ========================================================================= - def _get_or_create_encryption_bundles_folder_id(self) -> str: - """Get or create the encryption bundles folder for this user.""" + def _find_encryption_bundles_folder_id(self) -> str | None: + """The id of the encryption bundles folder of this user, or None when it does not exist.""" if self._encryption_bundles_folder_id: return self._encryption_bundles_folder_id folder_name = GdriveEncryptionBundlesFolder(email=self.email).as_string() - syftbox_folder_id = self.get_syftbox_folder_id() - folder_id = self._find_folder_by_name(folder_name, parent_id=syftbox_folder_id) + folder_id = self._find_folder_by_name( + folder_name, parent_id=self.get_syftbox_folder_id() + ) + if folder_id: + self._encryption_bundles_folder_id = folder_id + return folder_id + + def _get_or_create_encryption_bundles_folder_id(self) -> str: + """Get or create the encryption bundles folder for this user.""" + folder_id = self._find_encryption_bundles_folder_id() if not folder_id: - folder_id = self.create_folder(folder_name, syftbox_folder_id) - self._encryption_bundles_folder_id = folder_id + folder_name = GdriveEncryptionBundlesFolder(email=self.email).as_string() + folder_id = self.create_folder(folder_name, self.get_syftbox_folder_id()) + self._encryption_bundles_folder_id = folder_id return folder_id def _encryption_bundle_filename(self, owner_email: str, peer_email: str) -> str: @@ -2436,8 +2452,13 @@ def share_encryption_bundles_folder(self, peer_email: str) -> None: self.add_permission(folder_id, peer_email, write=False) def read_own_encryption_bundle(self, peer_email: str) -> str | None: - """Read the bundle this client published for ``peer_email``, or None.""" - folder_id = self._get_or_create_encryption_bundles_folder_id() + """Read the bundle this client published for ``peer_email``, or None. + + A read never creates the bundles folder: without it, nothing is published. + """ + folder_id = self._find_encryption_bundles_folder_id() + if folder_id is None: + return None filename = self._encryption_bundle_filename(self.email, peer_email) query = f"name='{filename}' and '{folder_id}' in parents and trashed=false" results = execute_with_retries( diff --git a/syft/sync/encryption_policy.py b/syft/sync/encryption_policy.py new file mode 100644 index 00000000000..a130a3e241f --- /dev/null +++ b/syft/sync/encryption_policy.py @@ -0,0 +1,64 @@ +"""Encryption policy that a client advertises to its peers.""" + +from enum import Enum +from pathlib import Path + + +class EncryptionPolicy(str, Enum): + """What a client asks of the encryption between it and a peer. + + - ``OPTIONAL``: encrypt only with a peer that requires it. + - ``REQUIRED``: encrypt with every peer, and refuse plaintext. + + A pair encrypts when one or both sides require it. + """ + + OPTIONAL = "optional" + REQUIRED = "required" + + @classmethod + def from_bool(cls, encryption: bool | None) -> "EncryptionPolicy | None": + """The policy for an ``encryption`` setting: ``True`` is ``REQUIRED``. + + ``None`` means no policy was asked for: the datasite keeps its policy. + + Raises: + TypeError: if ``encryption`` is not a bool or None. + """ + if encryption is None: + return None + if not isinstance(encryption, bool): + raise TypeError( + f"encryption must be True, False or None, got {encryption!r} " + f"({type(encryption).__name__})" + ) + return cls.REQUIRED if encryption else cls.OPTIONAL + + +class EncryptionPolicyMismatchError(ValueError): + """A client asked for another policy than the datasite uses.""" + + def __init__( + self, path: Path, on_disk: EncryptionPolicy, requested: EncryptionPolicy + ): + super().__init__( + f"This SyftBox uses encryption policy {on_disk.value!r} ({path}), " + f"and this client asked for {requested.value!r}. Every process on " + "a SyftBox uses the same policy. Log in without `encryption` to " + f"use {on_disk.value!r}, or delete the SyftBox " + "(client.delete_syftbox()) to start over with another policy." + ) + self.on_disk = on_disk + self.requested = requested + + +class MessageFormatError(ValueError): + """A message is plaintext in a pair that encrypts, or the other way around.""" + + def __init__(self, peer_email: str, encrypted: bool): + if encrypted: + detail = "is encrypted, and the pair does not encrypt" + else: + detail = "is not encrypted, and the pair encrypts" + super().__init__(f"A message from {peer_email} {detail}.") + self.peer_email = peer_email diff --git a/syft/sync/login.py b/syft/sync/login.py index e7a43d12363..d61ebf412e7 100644 --- a/syft/sync/login.py +++ b/syft/sync/login.py @@ -2,7 +2,8 @@ from syft.sync.utils.syftbox_utils import check_env from syft.sync.environments.environment import Environment -from syft.sync.syftbox_manager import SyftboxManager +from syft.sync.peers.peer_store import datasite_crypto_keys_path, import_key_file +from syft.sync.syftbox_manager import SyftboxManager, SyftboxManagerConfig from syft.sync.utils.print_utils import ( print_client_connected, print_client_connecting, @@ -42,7 +43,7 @@ def _init_client_login( # file at the version that first created it. Two things then break: the # login mismatch check reads that stale file and prompts at every login, and # a peer reads it to select a job or dataset protocol version for us. - client.peer_manager.write_own_version() + client.peer_manager.write_own_peer_info() if repair_pending: if sync: @@ -93,6 +94,19 @@ def _resolve_login_params( return email, token_path +def _import_key_file( + key_file: str | Path | None, syftbox_folder: Path, email: str +) -> None: + """Copy ``key_file``, the key file of an earlier session, into the datasite of ``email``. + + Call it before the client is built, so that the client loads these keys. + """ + if key_file is not None: + import_key_file( + key_file, datasite_crypto_keys_path(syftbox_folder, email), email + ) + + def login( email: str | None = None, sync: bool = True, @@ -100,6 +114,8 @@ def login( token_path: str | Path | None = None, skip_peer_on_patch_version_diff: bool | None = None, # None: value is determined by the role + encryption: bool | None = None, # True: required; None: the datasite's policy + key_file: str | Path | None = None, # crypto_keys.json of an earlier session ): return login_ds( email, @@ -107,6 +123,8 @@ def login( load_peers, token_path, skip_peer_on_patch_version_diff=skip_peer_on_patch_version_diff, + encryption=encryption, + key_file=key_file, ) @@ -117,6 +135,8 @@ def login_ds( token_path: str | Path | None = None, skip_peer_on_patch_version_diff: bool | None = None, # None: value is determined by the role + encryption: bool | None = None, # True: required; None: the datasite's policy + key_file: str | Path | None = None, # crypto_keys.json of an earlier session ): env = check_env() email, token_path = _resolve_login_params(email, token_path) @@ -124,18 +144,22 @@ def login_ds( repair_pending = handle_potential_version_mismatches_on_login(email, token_path) if env == Environment.COLAB: - client = SyftboxManager.for_colab( + config = SyftboxManagerConfig.for_colab( email=email, has_ds_role=True, skip_peer_on_patch_version_diff=skip_peer_on_patch_version_diff, + encryption=encryption, ) else: - client = SyftboxManager.for_jupyter( + config = SyftboxManagerConfig.for_jupyter( email=email, has_ds_role=True, - token_path=token_path, + token_path=Path(token_path) if token_path is not None else None, skip_peer_on_patch_version_diff=skip_peer_on_patch_version_diff, + encryption=encryption, ) + _import_key_file(key_file, config.syftbox_folder, email) + client = SyftboxManager.from_config(config) return _init_client_login(client, sync, load_peers, repair_pending) @@ -147,6 +171,8 @@ def login_do( token_path: str | Path | None = None, skip_peer_on_patch_version_diff: bool | None = None, # None: value is determined by the role + encryption: bool | None = None, # True: required; None: the datasite's policy + key_file: str | Path | None = None, # crypto_keys.json of an earlier session ): env = check_env() email, token_path = _resolve_login_params(email, token_path) @@ -154,17 +180,21 @@ def login_do( repair_pending = handle_potential_version_mismatches_on_login(email, token_path) if env == Environment.COLAB: - client = SyftboxManager.for_colab( + config = SyftboxManagerConfig.for_colab( email=email, has_do_role=True, skip_peer_on_patch_version_diff=skip_peer_on_patch_version_diff, + encryption=encryption, ) else: - client = SyftboxManager.for_jupyter( + config = SyftboxManagerConfig.for_jupyter( email=email, has_do_role=True, - token_path=token_path, + token_path=Path(token_path) if token_path is not None else None, skip_peer_on_patch_version_diff=skip_peer_on_patch_version_diff, + encryption=encryption, ) + _import_key_file(key_file, config.syftbox_folder, email) + client = SyftboxManager.from_config(config) return _init_client_login(client, sync, load_peers, repair_pending) diff --git a/syft/sync/login_utils.py b/syft/sync/login_utils.py index 86e5a0ad33b..ee757e0bea5 100644 --- a/syft/sync/login_utils.py +++ b/syft/sync/login_utils.py @@ -22,7 +22,8 @@ def _read_remote_version( from syft.sync.connections.drive.gdrive_transport import GDriveConnection conn = GDriveConnection.from_token_path(email=email, token_path=token_path) - return conn.read_own_version_file() + peer_info = conn.read_own_peer_info_file() + return peer_info.version_info if peer_info else None def _handle_version_incompatible( @@ -46,9 +47,9 @@ def _handle_version_incompatible( if choice == "1": print( f"Continuing with v{SYFT_VERSION}. Local and remote data are kept.\n" - "Encryption keys are the one exception. A key file from a newer " - "client is refused, because a private key cannot be rebuilt. Install " - "that client to use those keys.\n" + "Encryption keys are the one exception. A key file of another " + "version is refused, because a private key cannot be rebuilt. Install " + "the client that wrote it to use those keys.\n" ) return True if choice == "2": diff --git a/syft/sync/peers/peer.py b/syft/sync/peers/peer.py index 6ce4dc4a43b..2e031eb5cfb 100644 --- a/syft/sync/peers/peer.py +++ b/syft/sync/peers/peer.py @@ -1,7 +1,9 @@ from enum import Enum from typing import Any, List, Optional -from pydantic import BaseModel, PrivateAttr +from pydantic import BaseModel, Field, PrivateAttr +from syft.sync.encryption_policy import EncryptionPolicy from syft.sync.platforms.base_platform import BasePlatform +from syft.sync.version.peer_info import PeerInfo from syft.sync.version.version_info import VersionInfo @@ -24,13 +26,25 @@ class Peer(BaseModel): email: str platforms: List[BasePlatform] = [] state: PeerState = PeerState.ACCEPTED # Default for backward compatibility - version: Optional[VersionInfo] = None + info: Optional[PeerInfo] = None public_encryption_bundle: Optional[dict] = None - use_encryption: bool = False + # The state at the first load_peers() of this process, for a peer that was + # connected or waiting then. Known in this process only, so not saved. + state_at_start: Optional[PeerState] = Field(default=None, exclude=True) # Set by SyftboxManager.peers when this Peer is handed out. _manager: Any = PrivateAttr(default=None) + @property + def version(self) -> Optional[VersionInfo]: + """The version the peer advertises, or None before its version file is read.""" + return self.info.version_info if self.info else None + + @property + def encryption_policy(self) -> Optional[EncryptionPolicy]: + """The policy the peer advertises, or None before its version file is read.""" + return self.info.encryption_policy if self.info else None + @property def is_approved(self) -> bool: """Returns True if peer is accepted""" diff --git a/syft/sync/peers/peer_store.py b/syft/sync/peers/peer_store.py index 70df8a2384f..b4c38509c8d 100644 --- a/syft/sync/peers/peer_store.py +++ b/syft/sync/peers/peer_store.py @@ -1,16 +1,23 @@ """PeerStore: merged peer list + encryption key management. -Holds peers, encryption keys, and a use_encryption flag. +Holds peers, encryption keys, and the encryption policy of this client. Shared between PeerManager and all ConnectionRouter instances. """ -import json +import os +import tempfile from pathlib import Path -from typing import List, Optional +from typing import Any, List, Optional import syft_crypto_python as syc -from pydantic import BaseModel, PrivateAttr - +from pydantic import BaseModel, ConfigDict, Field, PrivateAttr +from syft_migration import ProtocolSchema + +from syft.sync.encryption_policy import ( + EncryptionPolicy, + EncryptionPolicyMismatchError, + MessageFormatError, +) from syft.sync.peers.exceptions import SyftPeerNotReadyError from syft.sync.peers.peer import Peer, PeerState @@ -20,11 +27,43 @@ # colliding, and delete_syftbox removes them together with the folder. PRIVATE_DIR_NAME = "private" CRYPTO_KEYS_FILENAME = "crypto_keys.json" +# Settings every process on a datasite shares, next to the key file. +SETTINGS_FILENAME = "syft_settings.json" + +# Format of the crypto key file. Raise it when the layout of the file changes. +# A file of any other version is refused. +CRYPTO_KEYS_VERSION = 2 + + +class _KeyFile(BaseModel): + """The private key file of one identity. It holds the only copy of its keys.""" + + model_config = ConfigDict(frozen=True, extra="forbid") -# Format of the crypto key file. Raise it when the layout of the file changes, -# and add a read path for every earlier version. A file with no version was -# written before the field, and is version 0. -CRYPTO_KEYS_VERSION = 1 + version: int + email: str + keys_jwk: dict[str, Any] + + @classmethod + def read(cls, path: Path, email: str) -> "_KeyFile": + """Read the key file of ``email`` at ``path``. + + Raises: + ValueError: if the file has another shape or version, or belongs to + another identity. + """ + key_file = cls.model_validate_json(path.read_text()) + if key_file.version != CRYPTO_KEYS_VERSION: + raise ValueError( + f"The crypto key file at {path} has version {key_file.version}, " + f"and this client reads only version {CRYPTO_KEYS_VERSION}." + ) + if key_file.email != email: + raise ValueError( + f"Encryption key file {path} belongs to {key_file.email!r}, " + f"not {email!r}" + ) + return key_file def datasite_crypto_keys_path(syftbox_folder: Path | str, email: str) -> Path: @@ -32,23 +71,75 @@ def datasite_crypto_keys_path(syftbox_folder: Path | str, email: str) -> Path: return Path(syftbox_folder) / email / PRIVATE_DIR_NAME / CRYPTO_KEYS_FILENAME +def datasite_settings_path(syftbox_folder: Path | str, email: str) -> Path: + """Per-datasite settings file: ``//private/syft_settings.json``.""" + return Path(syftbox_folder) / email / PRIVATE_DIR_NAME / SETTINGS_FILENAME + + +class _DatasiteSettings(BaseModel): + """The settings every process on one datasite shares.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + encryption_policy: EncryptionPolicy + + +def resolve_encryption_policy( + settings_path: Path | str, requested: EncryptionPolicy | None +) -> EncryptionPolicy: + """The encryption policy of the datasite with the settings file ``settings_path``. + + The first client to start writes ``requested`` (``OPTIONAL`` when None). + A later client gets the policy on disk. + + Raises: + EncryptionPolicyMismatchError: if ``requested`` differs from the + policy on disk. + """ + path = Path(settings_path) + if not path.exists(): + settings = _DatasiteSettings( + encryption_policy=requested or EncryptionPolicy.OPTIONAL + ) + try: + _create_private_file(path, settings.model_dump_json(indent=2)) + return settings.encryption_policy + except FileExistsError: + pass # Another process started first; its policy applies. + on_disk = _DatasiteSettings.model_validate_json(path.read_text()).encryption_policy + if requested is not None and requested != on_disk: + raise EncryptionPolicyMismatchError(path, on_disk, requested) + return on_disk + + class PeerStore(BaseModel): """Manages peers and encryption keys for E2E encryption.""" - model_config = {"arbitrary_types_allowed": True} + model_config = {"arbitrary_types_allowed": True, "extra": "forbid"} email: str - use_encryption: bool = False + encryption_policy: EncryptionPolicy = Field( + default=EncryptionPolicy.OPTIONAL, frozen=True + ) _private_keys: syc.SyftPrivateKeys | None = PrivateAttr(default=None) _peers: List[Peer] = PrivateAttr(default_factory=list) + @property + def requires_encryption(self) -> bool: + """True when this client requires encryption with every peer and at rest.""" + return self.encryption_policy == EncryptionPolicy.REQUIRED + # ========== Peer list methods ========== def clear_caches(self) -> None: """Clear the caches.""" self._peers = [] + @property + def peers(self) -> List[Peer]: + return list(self._peers) + @property def approved_peers(self) -> List[Peer]: return [p for p in self._peers if p.is_approved] @@ -72,17 +163,40 @@ def get_cached_peer(self, email: str) -> Optional[Peer]: return None def encrypt_if_needed(self, email: str, data: bytes) -> bytes: - if self.peer_uses_encryption(email): - return self.encrypt(email, data) - return data + """Encrypt ``data`` for ``email`` when the pair encrypts. + + Raises: + SyftPeerNotReadyError: when the policy of the peer is not known, or + the pair encrypts and the peer bundle is missing. + """ + if not self.pair_encrypts(self._peer_for_message(email)): + return data + return self.encrypt(email, data) def decrypt_and_verify_if_needed(self, email: str, data: bytes) -> bytes: - if self.peer_uses_encryption(email): - self.verify_message(email, data) - return self.decrypt(email, data) - return data + """Verify and decrypt a message from ``email`` when the pair encrypts. + + The format of ``data`` must match the pair. Plaintext in a pair that + encrypts is refused, so that a peer or the storage provider cannot + downgrade the pair. - def _is_syc_envelope(self, data: bytes) -> bool: + Raises: + SyftPeerNotReadyError: when the policy of the peer is not known, or + the pair encrypts and the peer bundle is missing. + MessageFormatError: when ``data`` is plaintext and the pair + encrypts, or ``data`` is an envelope and the pair does not. + """ + encrypts = self.pair_encrypts(self._peer_for_message(email)) + encrypted = self.is_envelope(data) + if encrypted != encrypts: + raise MessageFormatError(email, encrypted) + if not encrypts: + return data + self.verify_message(email, data) + return self.decrypt(email, data) + + @staticmethod + def is_envelope(data: bytes) -> bool: """True if `data` is an SYC encryption envelope (vs plaintext).""" try: syc.parse_envelope(data) @@ -97,19 +211,52 @@ def decrypt_dataset_if_needed(self, owner_email: str, data: bytes) -> bytes: plaintext bytes are passed through instead of raising. Bytes that are an SYC envelope are still signature-verified and decrypted as usual. """ - if not self.peer_uses_encryption(owner_email): - return data - if not self._is_syc_envelope(data): + if not self.is_envelope(data): return data self.verify_message(owner_email, data) return self.decrypt(owner_email, data) - def peer_uses_encryption(self, email: str) -> bool: + def pair_encrypts(self, peer: Optional[Peer]) -> bool: + """Whether messages with ``peer`` are encrypted: one side requires it. + + No peer, or a peer whose policy is not known yet, counts as not + requiring it. + """ + return self.requires_encryption or ( + peer is not None and peer.encryption_policy == EncryptionPolicy.REQUIRED + ) + + def _peer_for_message(self, email: str) -> Peer: + """The peer ``email``, to send a message to or read one from. + + Raises: + SyftPeerNotReadyError: while the peer is not loaded, or while + whether the pair encrypts depends on a policy not known yet. + """ + peer = self.get_cached_peer(email) + if peer is None or ( + not self.requires_encryption and peer.encryption_policy is None + ): + raise SyftPeerNotReadyError( + email, + cause=f"The encryption policy of {email} is not known yet.", + remedy=( + "Peer with them and wait until they accept. Then run " + "client.sync() and send again." + ), + ) + return peer + + def protocol_schema( + self, email: str, protocol_name: str + ) -> Optional[ProtocolSchema]: + """The schema ``email`` advertises for ``protocol_name``, or None when not known.""" peer = self.get_cached_peer(email) - return peer is not None and peer.use_encryption + version = peer.version if peer is not None else None + # getattr: a peer's version may be a V1 object (no schemas attribute). + return (getattr(version, "protocol_schemas", None) or {}).get(protocol_name) def set_peer(self, peer: Peer) -> None: - peer.use_encryption = self.use_encryption for i, p in enumerate(self._peers): if p.email == peer.email: self._peers[i] = peer @@ -117,12 +264,20 @@ def set_peer(self, peer: Peer) -> None: self._peers.append(peer) def add_peer(self, peer: Peer) -> None: - peer.use_encryption = self.use_encryption self._peers.append(peer) def set_peers(self, peers: List[Peer]) -> None: - for p in peers: - p.use_encryption = self.use_encryption + """Replace the peers, keeping what this process learned of each. + + ``info`` and ``state_at_start`` come from reads in this process, not + from the peer list, so a reload would lose them. + """ + previous = {p.email: p for p in self._peers} + for peer in peers: + old = previous.get(peer.email) + if old is not None: + peer.info = peer.info or old.info + peer.state_at_start = peer.state_at_start or old.state_at_start self._peers = peers # ========== Ensure helpers ========== @@ -193,9 +348,6 @@ def _missing_bundle_reason(peer: Peer) -> tuple[str, str]: def generate_keys(self) -> None: self._private_keys = syc.SyftRecoveryKey.generate().derive_keys() - def has_my_keys(self) -> bool: - return self._private_keys is not None - @property def public_key(self) -> syc.SyftPublicKeyBundle: keys = self._ensure_private_keys() @@ -263,99 +415,108 @@ def decrypt_for_self(self, envelope: bytes) -> bytes: return syc.decrypt_message(self.email, keys, own_bundle, parsed) def encrypt_for_self_if_needed(self, data: bytes) -> bytes: - """Encrypt for self if encryption is enabled and keys are available.""" - if self.use_encryption and self.has_my_keys(): + """Encrypt for self if this client requires encryption.""" + if self.requires_encryption: return self.encrypt_for_self(data) return data def decrypt_and_verify_for_self_if_needed(self, data: bytes) -> bytes: - """Verify and decrypt self-encrypted data if encryption is enabled and keys are available.""" - if self.use_encryption and self.has_my_keys(): + """Verify and decrypt data for self if this client requires encryption.""" + if self.requires_encryption: self.verify_message_from_self(data) return self.decrypt_for_self(data) return data # ========== Persistence ========== - def save_keys(self, path: Path) -> None: - keys = self._ensure_private_keys() - data = { - "version": CRYPTO_KEYS_VERSION, - "email": self.email, - "keys_jwk": keys.to_jwks(), - "peer_bundles": { - peer.email: peer.public_encryption_bundle - for peer in self._peers - if peer.public_encryption_bundle is not None - }, - } - path = Path(path) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(data, indent=2)) + def save_keys(self, path: Path | str) -> None: + """Write our keys to a new key file at ``path``. - @classmethod - def load_keys(cls, path: Path) -> "PeerStore": - data = json.loads(Path(path).read_text()) - # A file with no version was written before the field, and its layout is - # this client reads. A later version is refused: a user cannot rebuild a - # private key, so a wrong read loses the keys. - version = data.get("version", 0) - if version > CRYPTO_KEYS_VERSION: - raise ValueError( - f"The crypto key file at {path} has version {version}, and this " - f"client reads up to version {CRYPTO_KEYS_VERSION}. Install a " - "newer syft to use these keys." - ) - store = cls(email=data["email"], use_encryption=True) - store._private_keys = syc.SyftPrivateKeys.from_jwks(data["keys_jwk"]) - for email, bundle_dict in data.get("peer_bundles", {}).items(): - peer = Peer( - email=email, - public_encryption_bundle=bundle_dict, - use_encryption=True, - ) - store._peers.append(peer) - return store + Raises: + FileExistsError: if a file exists at ``path``. + """ + keys = self._ensure_private_keys() + key_file = _KeyFile( + version=CRYPTO_KEYS_VERSION, email=self.email, keys_jwk=keys.to_jwks() + ) + _create_private_file(Path(path), key_file.model_dump_json(indent=2)) @classmethod def create( cls, email: str, - use_encryption: bool = False, - keys_path: Path | str | None = None, + keys_path: Path | str, + encryption_policy: EncryptionPolicy = EncryptionPolicy.OPTIONAL, ) -> "PeerStore": - """Build a PeerStore, loading or generating encryption keys when enabled. + """Build a PeerStore for ``email`` with the keys in its key file. - - encryption off: return a plain store (no keys). - - encryption on: ``keys_path`` is required — the participant's own - per-datasite key file (``//private/crypto_keys.json``, - see :func:`datasite_crypto_keys_path`). Load it when present, else - generate a fresh key pair and persist it there. + ``keys_path`` is the participant's own key file + (``//private/crypto_keys.json``, see + :func:`datasite_crypto_keys_path`). When the file exists, the store + loads its keys. Else the store generates keys and writes the file. Scoping keys to a per-datasite file lets several identities run on one machine (e.g. two data owners in the same notebook) without colliding, and ties key lifetime to the datasite folder. Raises: - ValueError: if encryption is on but ``keys_path`` is missing, or the - existing key file belongs to a different identity. + ValueError: if the key file has another shape or version, or belongs + to another identity. + FileExistsError: if another process created the key file while + this store generated its keys. """ - if not use_encryption: - return cls(email=email, use_encryption=False) - if keys_path is None: - raise ValueError("keys_path is required when use_encryption is True") + store = cls(email=email, encryption_policy=encryption_policy) path = Path(keys_path) if path.exists(): - store = cls.load_keys(path) - if store.email != email: - raise ValueError( - f"Encryption key file {path} belongs to {store.email!r}, " - f"not {email!r}" - ) - return store + key_file = _KeyFile.read(path, email) + store._private_keys = syc.SyftPrivateKeys.from_jwks(key_file.keys_jwk) else: - # write keys to passed path - store = cls(email=email, use_encryption=True) store.generate_keys() store.save_keys(path) - return store + return store + + +def import_key_file(source: Path | str, keys_path: Path | str, email: str) -> None: + """Copy the key file ``source`` of ``email`` to ``keys_path``. + + Use it to restore the keys of an earlier session, e.g. an uploaded key file + on Colab, before the client is built. When ``keys_path`` holds the same + keys already, nothing changes. + + Raises: + ValueError: if ``source`` has another shape or version, or belongs to + another identity. + FileExistsError: if ``keys_path`` holds other keys. + """ + source, keys_path = Path(source), Path(keys_path) + key_file = _KeyFile.read(source, email) + if keys_path.exists(): + if _KeyFile.read(keys_path, email) != key_file: + raise FileExistsError( + f"{keys_path} holds other keys than {source}. Remove it to " + "import the file, or log in without the file." + ) + return + _create_private_file(keys_path, key_file.model_dump_json(indent=2)) + + +def _create_private_file(path: Path, content: str) -> None: + """Write ``content`` to a new file at ``path`` in one step. + + Used for the key file, which holds the only copy of the private keys, and + the settings file. The data goes to a temporary file in the same folder + and reaches the disk. A hard link then gives it its name. The link fails + with ``FileExistsError`` when that name exists, so a write cut short + leaves no partial file, and a second process cannot replace the file of + the first. ``mkstemp`` makes the file readable by this user only. + """ + path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.", suffix=".tmp") + try: + with os.fdopen(fd, "w") as f: + f.write(content) + f.flush() + os.fsync(f.fileno()) + os.link(tmp, path) + finally: + os.unlink(tmp) diff --git a/syft/sync/syftbox_manager.py b/syft/sync/syftbox_manager.py index 9946a501517..acb76484692 100644 --- a/syft/sync/syftbox_manager.py +++ b/syft/sync/syftbox_manager.py @@ -26,7 +26,7 @@ from syft.sync.job_file_change_handler import JobFileChangeHandler from syft.sync.peers.peer import Peer, PeerNotReadyError from syft.sync.peers.peer_list import PeerList -from syft.sync.peers.peer_store import PeerStore +from syft.sync.encryption_policy import EncryptionPolicy from syft.sync.platforms.base_platform import BasePlatform from syft.sync.sync.caches.datasite_owner_cache import ( DataSiteOwnerEventCacheConfig, @@ -134,9 +134,8 @@ def for_colab( email: str, has_ds_role: bool = False, has_do_role: bool = False, - encryption: bool = False, + encryption: bool | None = None, persist_owner_state: bool = True, - crypto_keys_path: Path | None = None, skip_peer_on_patch_version_diff: Optional[ bool ] = None, # None: value is determined by the role @@ -191,8 +190,7 @@ def for_colab( connection_configs=connection_configs, has_do_role=has_do_role, has_ds_role=has_ds_role, - use_encryption=encryption, - crypto_keys_path=crypto_keys_path, + encryption_policy=EncryptionPolicy.from_bool(encryption), skip_peer_on_patch_version_diff=skip_peer_on_patch_version_diff, force_ignore_peer_version=force_ignore_peer_version, ) @@ -216,9 +214,8 @@ def for_jupyter( has_ds_role: bool = False, has_do_role: bool = False, token_path: Path | None = None, - encryption: bool = False, + encryption: bool | None = None, persist_owner_state: bool = True, - crypto_keys_path: Path | None = None, skip_peer_on_patch_version_diff: Optional[ bool ] = None, # None: value is determined by the role @@ -277,8 +274,7 @@ def for_jupyter( connection_configs=connection_configs, has_do_role=has_do_role, has_ds_role=has_ds_role, - use_encryption=encryption, - crypto_keys_path=crypto_keys_path, + encryption_policy=EncryptionPolicy.from_bool(encryption), skip_peer_on_patch_version_diff=skip_peer_on_patch_version_diff, force_ignore_peer_version=force_ignore_peer_version, ) @@ -306,6 +302,7 @@ def _base_config_for_testing( persist_owner_state: bool = True, check_versions: bool = False, collection_specs: list["CollectionSyncSpec"] | None = None, + encryption_policy: EncryptionPolicy = EncryptionPolicy.OPTIONAL, ): # No collection specs by default: the generic sync engine knows nothing # about datasets. syft-rds registers its dataset spec at initialization @@ -354,6 +351,7 @@ def _base_config_for_testing( force_ignore_peer_version=not check_versions, has_do_role=has_do_role, has_ds_role=has_ds_role, + encryption_policy=encryption_policy, ) return cls( @@ -490,14 +488,14 @@ class SyftboxManager(BaseModelCallbackMixin): "should_create_checkpoint", "try_create_checkpoint", "delete_syftbox", - "write_own_version", + "write_own_peer_info", ) def __dir__(self): return list(self._PUBLIC_API) # Version file IO lives in syft.sync.version.local_version, and - # `write_own_version` writes both the local and the remote file. A + # `write_own_peer_info` writes both the local and the remote file. A # local-only writer on the manager leaves the remote file stale, which is # the bug that made the login mismatch prompt repeat at every login. @@ -563,30 +561,13 @@ def from_config(cls, config: SyftboxManagerConfig): config=config, ) - # PeerManager.from_config built the (possibly key-bearing) peer store from - # peer_manager_config; share that single store across every syncer's router - # so encryption is consistent on all sync paths. - if peer_manager.peer_store.use_encryption: - manager_res._set_peer_store(peer_manager.peer_store) - - # Every router that sends peer-directed messages downgrades them to the - # peer's negotiated syft protocol, so each one gets the live map. - manager_res._set_peer_schemas(peer_manager.live_peer_schemas("syft")) + # PeerManager.from_config built the peer store from peer_manager_config; + # share that single store across every syncer's router so all sync paths + # see the same peers, peer policies and keys. + manager_res._set_peer_store(peer_manager.peer_store) return manager_res - def _set_peer_schemas(self, peer_schemas) -> None: - """Wire PeerManager's live syft schema map into all routers.""" - if self.datasite_owner_syncer: - self.datasite_owner_syncer.connection_router.set_peer_schemas(peer_schemas) - if self.datasite_watcher_syncer: - self.datasite_watcher_syncer.connection_router.set_peer_schemas( - peer_schemas - ) - self.datasite_watcher_syncer.datasite_watcher_cache.connection_router.set_peer_schemas( - peer_schemas - ) - def _set_peer_store(self, peer_store) -> None: """Wire shared peer_store into all connection routers.""" from syft.sync.peers.peer_store import PeerStore @@ -609,9 +590,8 @@ def for_colab( email: str, has_ds_role: bool = False, has_do_role: bool = False, - encryption: bool = False, + encryption: bool | None = None, persist_owner_state: bool = True, - crypto_keys_path: Path | None = None, skip_peer_on_patch_version_diff: Optional[ bool ] = None, # None: value is determined by the role @@ -624,7 +604,6 @@ def for_colab( has_do_role=has_do_role, encryption=encryption, persist_owner_state=persist_owner_state, - crypto_keys_path=crypto_keys_path, skip_peer_on_patch_version_diff=skip_peer_on_patch_version_diff, force_ignore_peer_version=force_ignore_peer_version, ) @@ -638,9 +617,8 @@ def for_jupyter( has_ds_role: bool = False, has_do_role: bool = False, token_path: Path | None = None, - encryption: bool = False, + encryption: bool | None = None, persist_owner_state: bool = True, - crypto_keys_path: Path | None = None, skip_peer_on_patch_version_diff: Optional[ bool ] = None, # None: value is determined by the role @@ -656,7 +634,6 @@ def for_jupyter( token_path=token_path, encryption=encryption, persist_owner_state=persist_owner_state, - crypto_keys_path=crypto_keys_path, skip_peer_on_patch_version_diff=skip_peer_on_patch_version_diff, force_ignore_peer_version=force_ignore_peer_version, ) @@ -709,8 +686,8 @@ def _pair_with_google_drive_testing_connection( # Write version files if version checking is enabled if check_versions: - sender_manager.peer_manager.write_own_version() - receiver_manager.peer_manager.write_own_version() + sender_manager.peer_manager.write_own_peer_info() + receiver_manager.peer_manager.write_own_peer_info() # this makes sure that when we write a file as sender, the inactive file watcher picks it up sender_manager.file_writer.add_callback( @@ -766,6 +743,8 @@ def pair_with_mock_drive_service_connection( check_versions: bool = False, encryption: bool = False, collection_specs: list["CollectionSyncSpec"] | None = None, + ds_encryption_policy: EncryptionPolicy | None = None, + do_encryption_policy: EncryptionPolicy | None = None, ): """Create a pair of managers using mock Google Drive services for testing. @@ -783,10 +762,14 @@ def pair_with_mock_drive_service_connection( use_in_memory_cache: Whether to use in-memory caches persist_owner_state: Whether the DO keeps restorable owner state check_versions: Whether to check protocol/client versions + encryption: Require encryption on both sides + ds_encryption_policy: Policy of the DS; overrides ``encryption`` + do_encryption_policy: Policy of the DO; overrides ``encryption`` Returns: Tuple of (ds_manager, do_manager) """ + default_policy = EncryptionPolicy.from_bool(encryption) # Create configs using the existing base config generator do_config = SyftboxManagerConfig._base_config_for_testing( email=email1, @@ -797,6 +780,7 @@ def pair_with_mock_drive_service_connection( persist_owner_state=persist_owner_state, check_versions=check_versions, collection_specs=collection_specs, + encryption_policy=do_encryption_policy or default_policy, ) ds_config = SyftboxManagerConfig._base_config_for_testing( @@ -808,6 +792,7 @@ def pair_with_mock_drive_service_connection( persist_owner_state=persist_owner_state, check_versions=check_versions, collection_specs=collection_specs, + encryption_policy=ds_encryption_policy or default_policy, ) # Create managers from configs @@ -836,13 +821,8 @@ def pair_with_mock_drive_service_connection( ) # Write version files - ds_manager.peer_manager.write_own_version() - do_manager.peer_manager.write_own_version() - - # Initialize encryption if requested - if encryption: - ds_manager._init_encrypted_peer_store() - do_manager._init_encrypted_peer_store() + ds_manager.peer_manager.write_own_peer_info() + do_manager.peer_manager.write_own_peer_info() if add_peers: # DS creates peer request @@ -856,12 +836,6 @@ def pair_with_mock_drive_service_connection( return ds_manager, do_manager - def _init_encrypted_peer_store(self) -> None: - """Initialize the encrypted peer store.""" - peer_store = PeerStore(email=self.email, use_encryption=True) - peer_store.generate_keys() - self._set_peer_store(peer_store) - def add_peer( self, peer_email: str, @@ -983,6 +957,7 @@ def sync( separate process may have updated the file. """ with self.sync_file_lock(): + self.peer_manager.check_encryption_policy() self.load_peers(force_download=force_download_peer_state) if self.has_do_role: peer_emails = [peer.email for peer in self.peer_manager.approved_peers] @@ -1268,9 +1243,6 @@ def _delete_local_dirs(self): for d in dirs_to_delete: if d.exists(): shutil.rmtree(d) - # Encryption keys live in //private/crypto_keys.json, - # so deleting the syftbox folder above already removes them — a fresh state - # regenerates a new identity. # ========================================================================= # CHECKPOINT METHODS diff --git a/syft/sync/sync/caches/datasite_watcher_cache.py b/syft/sync/sync/caches/datasite_watcher_cache.py index 1d2112d3482..3d9b96865b8 100644 --- a/syft/sync/sync/caches/datasite_watcher_cache.py +++ b/syft/sync/sync/caches/datasite_watcher_cache.py @@ -241,9 +241,13 @@ def sync_down_parallel( self, peer_email: str, executor: ThreadPoolExecutor, - download_fn: Callable[[str], FileChangeEventsMessage], + download_fn: Callable[[str], FileChangeEventsMessage | None], ) -> int: - """Sync with parallel file downloads. Returns the number of events applied.""" + """Sync with parallel file downloads. Returns the number of events applied. + + ``download_fn`` gets the id of one file. It returns None for a message + that it skips. + """ peer_timestamp = self.last_event_timestamp_per_peer.get(peer_email) # Get file metadata (no download yet) @@ -259,7 +263,9 @@ def sync_down_parallel( # Download all files in parallel file_ids = [m["file_id"] for m in file_metadatas] - downloaded_messages = list(executor.map(download_fn, file_ids)) + downloaded_messages = [ + m for m in executor.map(download_fn, file_ids) if m is not None + ] # Apply in timestamp order event_count = 0 diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index 707a12da458..1a80444fff1 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -1,4 +1,5 @@ import logging +import warnings from pathlib import Path from uuid import uuid4 from functools import partial @@ -20,6 +21,7 @@ DataSiteOwnerEventCacheConfig, ) from syft.sync.connections.connection_router import ConnectionRouter +from syft.sync.peers.exceptions import SyftPeerNotReadyError from syft.sync.sync.caches.datasite_owner_cache import DataSiteOwnerEventCache from syft.sync.sync.collection_spec import CollectionSyncSpec from syft.sync.callback_mixin import BaseModelCallbackMixin @@ -45,6 +47,7 @@ logger = logging.getLogger(__name__) + # Default threshold for creating incremental checkpoint from rolling state DEFAULT_CHECKPOINT_EVENT_THRESHOLD = 50 @@ -226,14 +229,17 @@ def sync( # first, pull existing state for peer_email in peer_emails: proposed_count = 0 - while True: - msg = self.pull_and_process_next_proposed_filechange( - peer_email, raise_on_none=False - ) - if msg is None: - # no new message, we are done - break - proposed_count += len(msg.proposed_file_changes) + try: + while True: + msg = self.pull_and_process_next_proposed_filechange( + peer_email, raise_on_none=False + ) + if msg is None: + # no new message, we are done + break + proposed_count += len(msg.proposed_file_changes) + except SyftPeerNotReadyError as e: + warnings.warn(f"Messages from {peer_email} stay in the inbox: {e}") if proposed_count: print( f"Received {proposed_count} proposed change(s) " @@ -876,13 +882,15 @@ def process_syftbox_events_queue(self): file_change_events_message ) while not self.outbox_queue.empty(): - recipient, file_change_events_message = self.outbox_queue.get() + recipient, message = self.outbox_queue.get() if recipient in self._deferred_peer_emails: continue + # A send that cannot go raises before the clear, so the paths stay + # pending and a later sync sends them. self.connection_router.owner_write_event_messages_to_outbox( - recipient, file_change_events_message + recipient, message ) - self._clear_delivered(recipient, file_change_events_message) + self._clear_delivered(recipient, message) def _download_outbox_message_with_new_connection( self, file_id: str @@ -906,7 +914,8 @@ def compact_outbox_if_needed( if no outbox folder, or if peer encryption is on — DO can't read peer-encrypted bytes back to merge). """ - if self.connection_router.peer_store.peer_uses_encryption(recipient_email): + peer_store = self.connection_router.peer_store + if peer_store.pair_encrypts(peer_store.get_cached_peer(recipient_email)): logger.debug( f"outbox compaction for {recipient_email} skipped — " f"peer encryption is not supported yet" diff --git a/syft/sync/sync/datasite_watcher_syncer.py b/syft/sync/sync/datasite_watcher_syncer.py index 912f06c7d81..0e846750958 100644 --- a/syft/sync/sync/datasite_watcher_syncer.py +++ b/syft/sync/sync/datasite_watcher_syncer.py @@ -1,7 +1,7 @@ from pathlib import Path from concurrent.futures import ThreadPoolExecutor from queue import Queue -from typing import List +from typing import Dict, List from pydantic import BaseModel, Field, PrivateAttr @@ -9,6 +9,7 @@ from syft.sync.connections.connection_router import ConnectionRouter from syft.sync.callback_mixin import BaseModelCallbackMixin from syft.sync.events.file_change_event import FileChangeEventsMessage +from syft.sync.peers.exceptions import SyftPeerNotReadyError from syft.sync.messages.proposed_filechange import ( ProposedFileChange, ProposedFileChangesMessage, @@ -74,7 +75,8 @@ def get_proposed_file_change_object( ) def process_file_changes_queue(self): - file_changes = [] + # One message per datasite: each change goes to the owner of its path. + changes_by_recipient: Dict[str, List[ProposedFileChange]] = {} while not self.queue.empty(): relative_path, content = self.queue.get() @@ -97,22 +99,56 @@ def process_file_changes_queue(self): file_change = self.get_proposed_file_change_object( path_in_datasite, content, datasite_email=recipient ) - file_changes.append(file_change) + changes_by_recipient.setdefault(recipient, []).append(file_change) + self._send_by_recipient(changes_by_recipient) + + def _send_by_recipient( + self, changes_by_recipient: dict[str, list[ProposedFileChange]] + ) -> None: + """Send one message to each recipient. + + The changes for a recipient that cannot receive now are dropped. The + other recipients still get their changes, and then the first error is + raised. + + Raises: + SyftPeerNotReadyError: when the policy of a recipient is not known, + or the pair encrypts and the peer bundle is missing. + """ + error = None + for recipient, changes in changes_by_recipient.items(): + message = ProposedFileChangesMessage( + sender_email=self.email, proposed_file_changes=changes + ) + try: + self._send_message(recipient, message) + except SyftPeerNotReadyError as e: + error = error or e + if error is not None: + raise error + + def _record_sent_hashes(self, message: ProposedFileChangesMessage) -> None: + hashes = self.datasite_watcher_cache.file_hashes + for fc in message.proposed_file_changes: + path_key = Path(fc.datasite_email) / fc.path_in_datasite + if fc.is_deleted: + hashes.pop(path_key, None) + else: + hashes[path_key] = fc.new_hash - message = ProposedFileChangesMessage( - sender_email=self.email, proposed_file_changes=file_changes - ) + def _send_message( + self, recipient: str, message: ProposedFileChangesMessage + ) -> None: + """Send ``message`` and record the hashes it sent. + + Raises: + SyftPeerNotReadyError: when the policy of ``recipient`` is not + known, or the pair encrypts and the peer bundle is missing. + """ self.connection_router.watcher_send_proposed_file_changes_message( recipient, message ) - - # Update local cache with new hashes - for fc in file_changes: - path_key = Path(fc.datasite_email) / fc.path_in_datasite - if fc.is_deleted: - self.datasite_watcher_cache.file_hashes.pop(path_key, None) - else: - self.datasite_watcher_cache.file_hashes[path_key] = fc.new_hash + self._record_sent_hashes(message) def on_file_change( self, relative_path: Path | str, content: str | None = None, process_now=True @@ -126,14 +162,15 @@ def on_file_change( def download_events_message_with_new_connection( self, file_id: str, peer_email: str - ) -> FileChangeEventsMessage: - """Download from outbox using a new connection (thread-safe).""" + ) -> FileChangeEventsMessage | None: + """Download from outbox using a new connection (thread-safe). + + Returns None when the message is skipped (see + ``ConnectionRouter.watcher_read_events_message``). + """ connection = self.connection_router.connection_for_parallel_download() raw = connection.download_file(file_id) - raw = self.connection_router.peer_store.decrypt_and_verify_if_needed( - peer_email, raw - ) - return FileChangeEventsMessage.from_compressed_data(raw) + return self.connection_router.watcher_read_events_message(peer_email, raw) def download_collection_file_with_new_connection( self, file_id: str, owner_email: str diff --git a/syft/sync/transport/protocols.py b/syft/sync/transport/protocols.py index f8b4ec2e62e..2bd4d7644ec 100644 --- a/syft/sync/transport/protocols.py +++ b/syft/sync/transport/protocols.py @@ -23,7 +23,7 @@ if TYPE_CHECKING: from syft.sync.connections.base_connection import FileCollection - from syft.sync.version.version_info import VersionInfo + from syft.sync.version.peer_info import PeerInfo class P2PConnection(ABC): @@ -279,20 +279,23 @@ def read_peer_key_bundle(self, peer: str) -> str | None: # -- version documents ----------------------------------------------------- @abstractmethod - def publish_version(self, version_info: VersionInfo) -> None: + def publish_peer_info(self, peer_info: PeerInfo) -> None: """Publish my version document.""" @abstractmethod - def read_own_version(self) -> VersionInfo | None: + def read_own_peer_info(self) -> PeerInfo | None: """My published version document, or None.""" @abstractmethod - def read_peer_version(self, peer: str) -> VersionInfo | None: + def read_peer_info(self, peer: str) -> PeerInfo | None: """``peer``'s published version document, or None.""" @abstractmethod - def grant_version_read(self, peer: str) -> None: - """Let ``peer`` read my version document.""" + def grant_peer_info_read(self, peer: str, peer_info: PeerInfo) -> None: + """Let ``peer`` read my version document. + + When I have no document yet, publish ``peer_info`` first. + """ class Transport(BaseModel): diff --git a/syft/sync/utils/syftbox_utils.py b/syft/sync/utils/syftbox_utils.py index 8b7cbe1fd8a..b96c34cf53f 100644 --- a/syft/sync/utils/syftbox_utils.py +++ b/syft/sync/utils/syftbox_utils.py @@ -1,4 +1,5 @@ import random +import uuid import io import shutil import tarfile @@ -56,7 +57,11 @@ def random_email(): def random_syftbox_folder_for_testing(): - return Path(f"/tmp/sb_folder_testing-{random.randint(1, 1000000)}") + # Not ``random``: pytest-randomly reseeds it for each test, so two tests + # in one worker would share a folder, and its on-disk caches. Resolved, + # because /tmp links to /private/tmp on macOS, and code that resolves a + # path would otherwise not match the folder of the client. + return Path(f"/tmp/sb_folder_testing-{uuid.uuid4().hex}").resolve() def compress_data(data: bytes) -> bytes: diff --git a/syft/sync/version/peer_info.py b/syft/sync/version/peer_info.py new file mode 100644 index 00000000000..9bd7cb8f2a9 --- /dev/null +++ b/syft/sync/version/peer_info.py @@ -0,0 +1,55 @@ +""" +PeerInfo model: what a client publishes to its peers in SYFT_version.json. +""" + +from __future__ import annotations + +import json + +from syft_migration import MigratableObject + +from syft.migrations import client_registry, load_as_latest +from syft.sync.encryption_policy import EncryptionPolicy +from syft.sync.version.version_info import VersionInfo + +CANONICAL_NAME = "PeerInfo" + + +class PeerInfoV1(MigratableObject, registry=client_registry): + """The version of a client, and the settings its peers need from it. + + Stored as SYFT_version.json in the peer-visible SyftBox folder. VersionInfo + stays about the syft version only, so it can also serve as the local version + marker and in the release artifacts. + """ + + canonical_name: str = CANONICAL_NAME + version: str = "1" + + version_info: VersionInfo + # Peers read it to select encryption for the pair. + encryption_policy: EncryptionPolicy + + def to_json(self) -> str: + """Serialize to JSON string.""" + return self.model_dump_json(indent=2) + + @classmethod + def from_json(cls, json_str: str) -> "PeerInfo": + """Deserialize from JSON string, upgraded to the latest version. + + A syft older than PeerInfo wrote a bare VersionInfo. It advertises no + policy, so it reads as ``OPTIONAL``: the pair still encrypts when this + client requires it. + """ + data = json.loads(json_str) + if data.get("canonical_name") == CANONICAL_NAME: + return load_as_latest(data, CANONICAL_NAME) + return cls( + version_info=load_as_latest(data, "VersionInfo"), + encryption_policy=EncryptionPolicy.OPTIONAL, + ) + + +# Current-version alias: callers always work with the latest PeerInfo. +PeerInfo = PeerInfoV1 diff --git a/syft/sync/version/peer_manager.py b/syft/sync/version/peer_manager.py index db446817ac2..bab175cbfcc 100644 --- a/syft/sync/version/peer_manager.py +++ b/syft/sync/version/peer_manager.py @@ -11,12 +11,16 @@ from typing import Dict, List, Optional from pydantic import BaseModel, ConfigDict, PrivateAttr, model_validator -from syft_migration import ProtocolSchema - from syft.sync.connections.base_connection import ConnectionConfig from syft.sync.connections.connection_router import ConnectionRouter +from syft.sync.encryption_policy import EncryptionPolicy from syft.sync.peers.peer import Peer, PeerNotReadyError, PeerSetupError, PeerState -from syft.sync.peers.peer_store import PeerStore, datasite_crypto_keys_path +from syft.sync.peers.peer_store import ( + PeerStore, + datasite_crypto_keys_path, + datasite_settings_path, + resolve_encryption_policy, +) from syft.sync.utils.print_utils import ( print_peer_already_connected, print_peer_connection_established, @@ -29,6 +33,7 @@ VersionMismatchError, VersionUnknownError, ) +from syft.sync.version.peer_info import PeerInfo from syft.sync.version.version_info import CompatibilityStatus, VersionInfo logger = logging.getLogger(__name__) @@ -96,6 +101,8 @@ def raise_on_skip(self, operation: str) -> None: class PeerManagerConfig(BaseModel): """Configuration for PeerManager.""" + model_config = ConfigDict(extra="forbid") + syftbox_folder: Path email: str = "" connection_configs: List[ConnectionConfig] = [] @@ -106,24 +113,22 @@ class PeerManagerConfig(BaseModel): n_threads: int = 10 has_do_role: bool = False has_ds_role: bool = False - use_encryption: bool = False - # Explicit override for the encryption key file. When None, falls back to - # ``default_crypto_keys_path`` (the per-datasite location). - crypto_keys_path: Path | None = None + # None: the policy of the datasite on disk, or OPTIONAL on a new datasite. + encryption_policy: Optional[EncryptionPolicy] = None skip_peer_on_patch_version_diff: Optional[bool] = ( None # None: value is determined by the role (resolves to has_do_role) ) @property - def default_crypto_keys_path(self) -> Path: - """Default per-datasite encryption key file for this participant + def crypto_keys_path(self) -> Path: + """Per-datasite encryption key file for this participant (``//private/crypto_keys.json``).""" return datasite_crypto_keys_path(self.syftbox_folder, self.email) @property - def resolved_crypto_keys_path(self) -> Path: - """The explicit ``crypto_keys_path`` override if given, else the default.""" - return self.crypto_keys_path or self.default_crypto_keys_path + def settings_path(self) -> Path: + """Per-datasite settings file, shared by every process on the datasite.""" + return datasite_settings_path(self.syftbox_folder, self.email) @model_validator(mode="after") def _default_skip_peer_on_patch(self) -> "PeerManagerConfig": @@ -149,20 +154,8 @@ class PeerManager(BaseModel): _own_version: Optional[VersionInfo] = PrivateAttr(default=None) _executor: Optional[ThreadPoolExecutor] = PrivateAttr(default=None) - # protocol name -> {peer email -> ProtocolSchema}. Inner dicts are handed - # out by live_peer_schemas() and mutated in place as peer versions load, - # so consumers (JobStorage, DatasetStorage) always see current knowledge. - _peer_protocol_schemas: Dict[str, Dict[str, ProtocolSchema]] = PrivateAttr( - default_factory=dict - ) - # peer email -> last loaded VersionInfo (or None); lets a protocol map - # registered AFTER versions were loaded backfill instead of starting empty. - _loaded_peer_versions: Dict[str, Optional[VersionInfo]] = PrivateAttr( - default_factory=dict - ) - # Peer email -> state at the first load_peers(), for accepted peers and - # waiting requests. None until that load. - _states_at_start: Optional[Dict[str, PeerState]] = PrivateAttr(default=None) + # Set by the first load_peers(), which records Peer.state_at_start. + _first_load_done: bool = PrivateAttr(default=False) # ========== Peer List Properties ========== @@ -200,10 +193,10 @@ def from_config(cls, config: PeerManagerConfig, email: str = "") -> "PeerManager resolved_email = config.email or email peer_store = PeerStore.create( email=resolved_email, - use_encryption=config.use_encryption, - keys_path=config.resolved_crypto_keys_path - if config.use_encryption - else None, + encryption_policy=resolve_encryption_policy( + config.settings_path, config.encryption_policy + ), + keys_path=config.crypto_keys_path, ) connection_router = ConnectionRouter.from_configs( resolved_email, config.connection_configs @@ -227,131 +220,93 @@ def model_post_init(self, __context) -> None: """Initialize the thread pool executor.""" self._executor = ThreadPoolExecutor(max_workers=self.n_threads) - def live_peer_schemas(self, protocol_name: str) -> Dict[str, ProtocolSchema]: - """The live {peer email -> ProtocolSchema} map for ``protocol_name``. - - The returned dict is owned by this PeerManager and updated in place - whenever a peer's version file is (re)loaded: peers advertising the - protocol appear, peers that stop advertising it (or whose version is - cleared) disappear. Hand it to JobStorage/DatasetStorage as - ``peer_schemas`` so negotiation always uses current knowledge. A map - registered after peer versions were already loaded is backfilled from - them. - """ - per_peer = self._peer_protocol_schemas.get(protocol_name) - if per_peer is None: - per_peer = self._peer_protocol_schemas[protocol_name] = {} - for email, version_info in list(self._loaded_peer_versions.items()): - self._sync_one(per_peer, protocol_name, email, version_info) - return per_peer - - @staticmethod - def _sync_one( - per_peer: Dict[str, ProtocolSchema], - protocol_name: str, - peer_email: str, - version_info: Optional[VersionInfo], - ) -> None: - # getattr: a peer's version may be a V1 object (no schemas attribute). - advertised = getattr(version_info, "protocol_schemas", None) or {} - schema = advertised.get(protocol_name) - if schema is not None: - per_peer[peer_email] = schema - else: - per_peer.pop(peer_email, None) - - def _update_peer_schemas( - self, peer_email: str, version_info: Optional[VersionInfo] - ) -> None: - """Sync the live schema maps with a freshly loaded peer version. - - A None version or a pre-V2 version file (empty ``protocol_schemas``) - removes the peer: it is an unknown speaker, and consumers apply their - own unknown-peer defaults. - """ - self._loaded_peer_versions[peer_email] = version_info - # list(): guard against a concurrent live_peer_schemas() registration. - for protocol_name, per_peer in list(self._peer_protocol_schemas.items()): - self._sync_one(per_peer, protocol_name, peer_email, version_info) - def get_own_version(self) -> VersionInfo: """Get current client's version info.""" if self._own_version is None: self._own_version = VersionInfo.current() return self._own_version - def write_own_version(self) -> None: - """Write version file to own SyftBox folder (remote) and local disk.""" + def get_own_peer_info(self) -> PeerInfo: + """What this client publishes to its peers.""" + return PeerInfo( + version_info=self.get_own_version(), + encryption_policy=self.peer_store.encryption_policy, + ) + + def write_own_peer_info(self) -> None: + """Write the peer info file to own SyftBox folder, and the version to local disk.""" from syft.sync.version.local_version import write_local_version - version_info = self.get_own_version() - self.connection_router.write_version_file(version_info) + self.connection_router.write_peer_info_file(self.get_own_peer_info()) write_local_version(self.syftbox_folder) - def share_version_with_peer(self, peer_email: str) -> None: - """Share version file with a peer so they can read it.""" - self.connection_router.share_version_file_with_peer(peer_email) + def share_peer_info_with_peer(self, peer_email: str) -> None: + """Share the peer info file with a peer so they can read it.""" + self.connection_router.share_peer_info_file_with_peer( + peer_email, self.get_own_peer_info() + ) def get_cached_peer(self, email: str) -> Optional[Peer]: """Get a peer by email, or None if not found.""" return self.peer_store.get_cached_peer(email) - def load_peer_version(self, peer_email: str) -> Optional[VersionInfo]: - """Load version for a single peer (blocking).""" - version_info = self.connection_router.read_peer_version_file(peer_email) + def load_peer_info(self, peer_email: str) -> Optional[PeerInfo]: + """Load the info of a single peer (blocking).""" + peer_info = self.connection_router.read_peer_info_file(peer_email) + self._set_peer_info(peer_email, peer_info) + return peer_info + + def _set_peer_info(self, peer_email: str, peer_info: Optional[PeerInfo]) -> None: + """Cache a freshly loaded peer info on the peer. + + A read that finds nothing keeps the info known before: a peer that + deleted its SyftBox still sends messages under its last policy, such + as the deletes of its files. + """ cached_peer = self.get_cached_peer(peer_email) - if cached_peer: - cached_peer.version = version_info - self._update_peer_schemas(peer_email, version_info) - return version_info - - def _load_single_peer_version( - self, peer_email: str - ) -> tuple[str, Optional[VersionInfo]]: - """Load version for a single peer. Used by parallel loader.""" + if cached_peer and peer_info is not None: + cached_peer.info = peer_info + + def _load_single_peer_info(self, peer_email: str) -> tuple[str, Optional[PeerInfo]]: + """Load the info of a single peer. Used by parallel loader.""" try: # Create new connection for thread safety - connection = self.connection_router.connection_for_version_read( + connection = self.connection_router.connection_for_peer_file_read( create_new=True ) - version_info = connection.read_peer_version_file(peer_email) - return (peer_email, version_info) + peer_info = connection.read_peer_info_file(peer_email) except Exception as e: if not self.suppress_version_warnings: - warnings.warn(f"Failed to load version for {peer_email}: {e}") + warnings.warn(f"Failed to load peer info for {peer_email}: {e}") return (peer_email, None) + return (peer_email, peer_info) - def load_peer_versions_parallel( + def load_peer_infos_parallel( self, peer_emails: List[str], force: bool = False - ) -> Dict[str, Optional[VersionInfo]]: - """Load versions for multiple peers in parallel using internal executor.""" + ) -> Dict[str, Optional[PeerInfo]]: + """Load the infos of multiple peers in parallel using internal executor.""" if not peer_emails: return {} # Submit all tasks and collect results - results = list(self._executor.map(self._load_single_peer_version, peer_emails)) + results = list(self._executor.map(self._load_single_peer_info, peer_emails)) - # Update peer version fields and return - for email, version in results: - peer = self.get_cached_peer(email) - if peer: - peer.version = version - self._update_peer_schemas(email, version) + for email, peer_info in results: + self._set_peer_info(email, peer_info) - return {email: version for email, version in results} + return dict(results) def get_peer_version(self, peer_email: str) -> Optional[VersionInfo]: """Get cached version for peer, None if not loaded or not available.""" peer = self.get_cached_peer(peer_email) return peer.version if peer else None - def clear_peer_version(self, peer_email: str) -> None: - """Clear the cached version for a peer (useful for testing).""" - peer = self.get_cached_peer(peer_email) - if peer: - peer.version = None - self._update_peer_schemas(peer_email, None) + def clear_peer_info(self, peer_email: str) -> None: + """Clear the cached info of a peer (useful for testing).""" + cached_peer = self.get_cached_peer(peer_email) + if cached_peer: + cached_peer.info = None def peer_compatibility_status(self, peer_email: str) -> CompatibilityStatus: """Get the CompatibilityStatus for a peer (UNKNOWN if version not loaded).""" @@ -550,23 +505,25 @@ def add_peer(self, peer_email: str, force: bool = False, verbose: bool = True): if existing_peer_obj: new_peer_obj = existing_peer_obj - # Exchange encryption bundles if key_manager is set - if self.peer_store.use_encryption: - self._write_encryption_bundle_for_peer(peer_email) + # Both sides publish their bundle at peering, so a pair can start to + # encrypt in a later session. The peer policy selects encryption for + # the pair. Before the peer accepts, its version file is not shared + # and the policy is unknown; load_peers reads the bundle once it is. + self._write_encryption_bundle_for_peer(peer_email) + new_peer_obj.info = self.connection_router.read_peer_info_file(peer_email) + encrypted = self.peer_store.pair_encrypts(new_peer_obj) peer_bundle = None - if self.peer_store.use_encryption and is_accepting: + if encrypted and is_accepting: peer_bundle = self._read_peer_encryption_bundle(peer_email) self.connection_router.update_peer_state( peer_email, new_state.value, public_encryption_bundle=peer_bundle ) - self.share_version_with_peer(peer_email) - version_info = self.connection_router.read_peer_version_file(peer_email) - self._update_peer_schemas(peer_email, version_info) + self.share_peer_info_with_peer(peer_email) - new_peer_obj.version = version_info - new_peer_obj.public_encryption_bundle = peer_bundle + if peer_bundle is not None: + new_peer_obj.public_encryption_bundle = peer_bundle new_peer_obj.state = new_state self.peer_store.set_peer(new_peer_obj) @@ -577,9 +534,7 @@ def add_peer(self, peer_email: str, force: bool = False, verbose: bool = True): print_peer_request_sent(peer_email) def _write_encryption_bundle_for_peer(self, peer_email: str) -> dict | None: - """Write own encryption bundle for a peer if encryption is enabled.""" - if not self.peer_store.use_encryption: - raise ValueError("Encryption is not enabled") + """Write own encryption bundle for a peer.""" bundle = self.peer_store.get_public_bundle() bundle_json = json.dumps({"public_encryption_bundle": bundle}) self.connection_router.write_encryption_bundle(peer_email, bundle_json) @@ -587,12 +542,16 @@ def _write_encryption_bundle_for_peer(self, peer_email: str) -> dict | None: return bundle def _read_peer_encryption_bundle(self, peer_email: str) -> dict | None: - """Read a peer's encryption bundle if available.""" + """The bundle ``peer_email`` published for us, or None when it has not published one. + + Raises: + ValueError: when the published bundle file is not valid JSON. + KeyError: when it holds no bundle. + """ bundle_json = self.connection_router.read_peer_encryption_bundle(peer_email) if not bundle_json: return None - data = json.loads(bundle_json) - return data.get("public_encryption_bundle") + return json.loads(bundle_json)["public_encryption_bundle"] def load_peers(self, force_download: bool = False): """Load peers: from JSON (accepted + requested_by_me) + new requests from folder scan. @@ -637,29 +596,82 @@ def load_peers(self, force_download: bool = False): email, PeerState.ACCEPTED.value ) - if self._states_at_start is None: - self._states_at_start = { - p.email: p.state - for p in peers - if p.state in (PeerState.ACCEPTED, PeerState.REQUESTED_BY_PEER) - } + first_load = not self._first_load_done + if first_load: + self._first_load_done = True + for p in peers: + if p.state in (PeerState.ACCEPTED, PeerState.REQUESTED_BY_PEER): + p.state_at_start = p.state self.peer_store.set_peers(peers) + self.load_peer_infos_parallel([peer.email for peer in peers]) + + # Peer policies are known only after the versions load. + encrypted = [ + p + for p in peers + if p.state in (PeerState.ACCEPTED, PeerState.REQUESTED_BY_ME) + and self.peer_store.pair_encrypts(p) + ] + if first_load: + # Only a pair that encrypts uses the bundle we published. + self._check_published_bundles([p.email for p in encrypted]) + self._read_missing_peer_bundles(encrypted) + + def _check_published_bundles(self, peer_emails: list[str]) -> None: + """Raise when a bundle this client published differs from its keys. + + A runtime without the key file of the account makes new keys, and + peers keep the bundle of the earlier keys. The keys do not change + while the process runs, so the first peer load checks once. + + Raises: + ValueError: when a published bundle carries other keys. + """ + own_bundle = self.peer_store.get_public_bundle() + published = self._executor.map(self._read_own_published_bundle, peer_emails) + for peer_email, bundle in zip(peer_emails, published): + if bundle is not None and bundle != own_bundle: + raise ValueError( + f"The encryption key published for {peer_email} is not the " + "key of this client: this account already has a key. Log in " + "again with the key file of this account." + ) + + def _read_own_published_bundle(self, peer_email: str) -> dict | None: + """The bundle this client published for ``peer_email``, or None (thread-safe).""" + connection = self.connection_router.connection_for_peer_file_read( + create_new=True + ) + bundle_json = connection.read_own_encryption_bundle(peer_email) + if bundle_json is None: + return None + return json.loads(bundle_json)["public_encryption_bundle"] + + def _read_missing_peer_bundles(self, peers: list[Peer]) -> None: + """Read the published bundle of every peer we hold none for.""" + for peer in peers: + if self.peer_store.has_peer_bundle(peer.email): + continue + bundle = self._read_peer_encryption_bundle(peer.email) + if bundle: + self.peer_store.set_peer_bundle(peer.email, bundle) + self.connection_router.update_peer_state( + peer.email, peer.state.value, public_encryption_bundle=bundle + ) - # Try to read encryption bundles from GDrive for peers missing bundles - if self.peer_store.use_encryption: - for peer in peers: - if peer.state in (PeerState.ACCEPTED, PeerState.REQUESTED_BY_ME): - if not self.peer_store.has_peer_bundle(peer.email): - bundle = self._read_peer_encryption_bundle(peer.email) - if bundle: - self.peer_store.set_peer_bundle(peer.email, bundle) - self.connection_router.update_peer_state( - peer.email, - peer.state.value, - public_encryption_bundle=bundle, - ) - - self.load_peer_versions_parallel([peer.email for peer in peers]) + def check_encryption_policy(self) -> None: + """Raise when the datasite policy on disk is not the policy of this client. + + Another process may have deleted the SyftBox and started with another + policy. A missing settings file gets the policy of this client. + + Raises: + EncryptionPolicyMismatchError: if the policy on disk differs. + """ + resolve_encryption_policy( + datasite_settings_path(self.syftbox_folder, self.peer_store.email), + self.peer_store.encryption_policy, + ) def validate_peer(self, peer_email: str) -> Peer: """Return the peer ``peer_email`` if it is approved, else raise. @@ -699,7 +711,8 @@ def validate_peer(self, peer_email: str) -> Peer: def _warn_if_known_at_start(self, peer_email: str) -> None: """Warn if ``peer_email`` was connected, or waiting, at the first load.""" - state = (self._states_at_start or {}).get(peer_email) + peer = self.get_cached_peer(peer_email) + state = peer.state_at_start if peer is not None else None if state is None: return found = ( @@ -718,7 +731,9 @@ def forget_states_at_start(self) -> None: Call it after this client deletes its state. Its peers then connect again from nothing, so ``validate_peer`` does not warn about them. """ - self._states_at_start = {} + self._first_load_done = True + for peer in self.peer_store.peers: + peer.state_at_start = None def peer_may_be_valid(self, peer_email: str) -> bool: """Tell if ``validate_peer`` can pass after the next ``load_peers()``. diff --git a/tests/conftest.py b/tests/conftest.py index bcb6491d3f8..08e93461760 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,6 +8,8 @@ import pytest +from syft.sync.peers.peer_store import PeerStore + @pytest.fixture(scope="session", autouse=True) def disable_pre_sync_for_tests(): @@ -31,3 +33,23 @@ def disable_pre_sync_for_tests(): os.environ["PRE_SYNC"] = original_value else: os.environ.pop("PRE_SYNC", None) + + +@pytest.fixture +def advertised_schemas(monkeypatch): + """Protocol schemas the test makes peers advertise. + + ``advertised_schemas[email, protocol_name] = schema`` replaces what the + peer info of ``email`` says, for every client in the test. ``None`` makes + the schema unknown. Use it to stand in for a peer on an older release. + """ + overrides = {} + read_from_peer_info = PeerStore.protocol_schema + + def protocol_schema(self, email, protocol_name): + if (email, protocol_name) in overrides: + return overrides[email, protocol_name] + return read_from_peer_info(self, email, protocol_name) + + monkeypatch.setattr(PeerStore, "protocol_schema", protocol_schema) + return overrides diff --git a/tests/integration/syft_bg/test_cli.py b/tests/integration/syft_bg/test_cli.py index bf4f4b3d21b..fd00bf05469 100644 --- a/tests/integration/syft_bg/test_cli.py +++ b/tests/integration/syft_bg/test_cli.py @@ -27,13 +27,16 @@ class TestInitCommand: def test_init_basic(self, mock_api_init): """Should pass email to api.init.""" runner = CliRunner() - result = runner.invoke(init, ["-e", "alice@uni.edu"]) + result = runner.invoke( + init, ["-e", "alice@uni.edu", "--encryption", "optional"] + ) assert result.exit_code == 0 mock_api_init.assert_called_once_with( do_email="alice@uni.edu", syftbox_root=None, token_path=None, + encryption=False, ) @patch("syft_bg.api.api.init") @@ -50,6 +53,15 @@ def test_init_all_options(self, mock_api_init): # unless file exists. Test the basic flow instead. mock_api_init.assert_not_called() # file doesn't exist + @patch("syft_bg.api.api.init") + def test_init_without_encryption_keeps_syftbox_policy(self, mock_api_init): + """Without --encryption, api.init gets None: the SyftBox keeps its policy.""" + runner = CliRunner() + result = runner.invoke(init, ["-e", "alice@uni.edu"]) + + assert result.exit_code == 0 + assert mock_api_init.call_args.kwargs["encryption"] is None + @patch("syft_bg.api.api.init") def test_init_requires_email(self, mock_api_init): """Should error when email not provided.""" @@ -386,7 +398,7 @@ class TestRemovePeerCommand: def test_remove_peer(self, api_store, temp_dir): """Should remove peer from apis.""" _create_api( - api_store, temp_dir, "my_analysis", ["main.py"], ["a@x.com", "b@x.com"] + api_store, temp_dir, "my_analysis", ["run.sh"], ["a@x.com", "b@x.com"] ) runner = CliRunner() @@ -398,7 +410,7 @@ def test_remove_peer(self, api_store, temp_dir): def test_removing_last_peer_asks_for_confirmation(self, api_store, temp_dir): """An object left without peers matches anyone, so ask first.""" - _create_api(api_store, temp_dir, "my_analysis", ["main.py"], ["a@x.com"]) + _create_api(api_store, temp_dir, "my_analysis", ["run.sh"], ["a@x.com"]) runner = CliRunner() result = runner.invoke(remove_peer, ["a@x.com"], input="n\n") diff --git a/tests/integration/with_unit_coverage/test_sync_manager.py b/tests/integration/with_unit_coverage/test_sync_manager.py index 1430b663c21..ab6fb0cd59a 100644 --- a/tests/integration/with_unit_coverage/test_sync_manager.py +++ b/tests/integration/with_unit_coverage/test_sync_manager.py @@ -541,8 +541,8 @@ def test_version_upgrade_breaks_communication(): sleep(2) # Verify initial version compatibility - ds_manager.peer_manager.load_peer_version(do_manager.email) - do_manager.peer_manager.load_peer_version(ds_manager.email) + ds_manager.peer_manager.load_peer_info(do_manager.email) + do_manager.peer_manager.load_peer_info(ds_manager.email) assert ds_manager.peer_manager.is_peer_version_compatible(do_manager.email), ( "DS should see DO as compatible initially" @@ -564,17 +564,20 @@ def test_version_upgrade_breaks_communication(): ) # Write the new version to GDrive (simulating DS upgrading their client) - ds_connection.write_version_file(new_version) + own_info = ds_manager.peer_manager.get_own_peer_info() + ds_connection.write_peer_info_file( + own_info.model_copy(update={"version_info": new_version}) + ) # Phase 3: Clear DO's cached version of DS and reload from GDrive - do_manager.peer_manager.clear_peer_version(ds_manager.email) + do_manager.peer_manager.clear_peer_info(ds_manager.email) # Reload DS's version (this fetches from GDrive) - reloaded_version = do_manager.peer_manager.load_peer_version(ds_manager.email) + reloaded_info = do_manager.peer_manager.load_peer_info(ds_manager.email) # Verify the new version was loaded - assert reloaded_version is not None, "Should be able to reload peer version" - assert reloaded_version.syft_client_version == "99.0.0", ( + assert reloaded_info is not None, "Should be able to reload peer version" + assert reloaded_info.version_info.syft_client_version == "99.0.0", ( "Reloaded version should be the upgraded version" ) @@ -584,4 +587,4 @@ def test_version_upgrade_breaks_communication(): ) # Cleanup: Restore DS's version file to the original version - ds_connection.write_version_file(current) + ds_connection.write_peer_info_file(own_info) diff --git a/tests/integration/without_unit_coverage/test_sync_manager.py b/tests/integration/without_unit_coverage/test_sync_manager.py index 4f1c01c6310..99388e6232c 100644 --- a/tests/integration/without_unit_coverage/test_sync_manager.py +++ b/tests/integration/without_unit_coverage/test_sync_manager.py @@ -5,10 +5,12 @@ cannot be adequately tested with mocked connections. """ +from syft.sync.peers.exceptions import SyftPeerNotReadyError from syft.sync.syftbox_manager import SyftboxManager +from syft.sync.utils.waiting import poll_until import os +import time from pathlib import Path -from time import sleep import pytest @@ -28,6 +30,17 @@ token_path_do = CREDENTIALS_DIR / FILE_DO token_path_ds = CREDENTIALS_DIR / FILE_DS +# Drive lists a new file some time after the write. +POLL_TIMEOUT = 60 +POLL_INTERVAL = 1 + + +def _poll(check, what: str): + """Call ``check`` until it returns a value that is not None.""" + result = poll_until(check, time.monotonic() + POLL_TIMEOUT, POLL_INTERVAL) + assert result is not None, f"Timed out after {POLL_TIMEOUT}s waiting for {what}" + return result + @pytest.mark.flaky(reruns=3, reruns_delay=2) @pytest.mark.usefixtures("setup_delete_syftboxes") @@ -37,10 +50,10 @@ def test_peer_request_blocks_sync_until_approved(): Workflow: 1. DS adds DO as peer (creates peer request) - 2. DS submits a job + 2. DS submits a job - the send raises (DO policy not known) 3. DO syncs - nothing should sync (peer not approved) 4. DO approves peer request - 5. DO syncs - job should now sync + 5. DS syncs and submits again, DO syncs - job should now sync """ # Create managers with Google Drive connection, no auto-add peers ds_manager, do_manager = SyftboxManager._pair_with_google_drive_testing_connection( @@ -59,11 +72,12 @@ def test_peer_request_blocks_sync_until_approved(): # Step 1: DS makes peer request by adding DO ds_manager.add_peer(do_manager.email) - # Wait for sync - sleep(1) - # Verify: DO sees this as a pending request - do_manager.load_peers() + def request_seen(): + do_manager.load_peers() + return do_manager.peer_manager.requested_by_peer_peers or None + + _poll(request_seen, "the peer request") assert len(do_manager.peer_manager.requested_by_peer_peers) == 1 assert len(do_manager.peer_manager.approved_peers) == 0 assert do_manager.peer_manager.requested_by_peer_peers[0].email == ds_manager.email @@ -71,10 +85,8 @@ def test_peer_request_blocks_sync_until_approved(): # Step 2: DS submits a simple job job_file_path = f"{do_manager.email}/test.job" job_content = "print('Hello from DS')" - ds_manager._send_file_change(job_file_path, job_content) - - # Wait for message to be sent - sleep(1) + with pytest.raises(SyftPeerNotReadyError): + ds_manager._send_file_change(job_file_path, job_content) # Step 3: DO syncs WITHOUT accepting - nothing should sync do_manager.sync() @@ -91,11 +103,24 @@ def test_peer_request_blocks_sync_until_approved(): assert len(do_manager.peer_manager.approved_peers) == 1 assert do_manager.peer_manager.approved_peers[0].email == ds_manager.email - # Step 5: DO syncs again - now it should work - do_manager.sync() + # Step 5: DS syncs, which reads the DO policy, and submits again. DO syncs + # again - now it should work + def resent(): + ds_manager.sync() + try: + ds_manager._send_file_change(job_file_path, job_content) + except SyftPeerNotReadyError: + return None # The DS does not see the DO version file yet. + return True + + _poll(resent, "the DO policy") + + def job_received(): + do_manager.sync() + return do_cache.file_hashes or None # Verify: File synced and in cache - assert len(do_cache.file_hashes) > 0, "Cache should have content after approval" + _poll(job_received, "the job at the DO") # Verify: File is tracked in cache with correct path (stored as PosixPath) expected_cache_path = Path("test.job") diff --git a/tests/migrations/p2p/test_dataset_multicopy_delivery.py b/tests/migrations/p2p/test_dataset_multicopy_delivery.py index 56fbed743b0..c3f174c0ef9 100644 --- a/tests/migrations/p2p/test_dataset_multicopy_delivery.py +++ b/tests/migrations/p2p/test_dataset_multicopy_delivery.py @@ -159,14 +159,12 @@ def test_a_dataset_for_a_protocol0_peer_arrives_flat_and_reads(pair): ) -def test_a_create_for_a_protocol0_peer_publishes_both_layouts(pair): +def test_a_create_for_a_protocol0_peer_publishes_both_layouts(pair, advertised_schemas): # The create writes the current layout always, and the layout of each peer # in the audience. A peer that reads only the older one still finds a copy, # and a peer that reads both prefers the newest. ds_manager, do_manager = pair - do_manager.peer_manager.live_peer_schemas("syft-dataset")[OLD_PEER] = ( - _dataset_schema("0") - ) + advertised_schemas[OLD_PEER, "syft-dataset"] = _dataset_schema("0") mock_path, private_path, readme_path = create_tmp_dataset_files() do_manager.create_dataset( @@ -232,11 +230,11 @@ def test_the_owner_listing_names_each_dataset_once(pair): assert tags.count("mixed dataset") == 1 -def _create_for_a_mixed_audience(ds_manager, do_manager, name: str, **kwargs): +def _create_for_a_mixed_audience( + advertised_schemas, ds_manager, do_manager, name: str, **kwargs +): """Create a dataset for an audience of one protocol-0 peer and one current peer.""" - do_manager.peer_manager.live_peer_schemas("syft-dataset")[OLD_PEER] = ( - _dataset_schema("0") - ) + advertised_schemas[OLD_PEER, "syft-dataset"] = _dataset_schema("0") mock_path, private_path, readme_path = create_tmp_dataset_files() return do_manager.create_dataset( name=name, @@ -248,29 +246,35 @@ def _create_for_a_mixed_audience(ds_manager, do_manager, name: str, **kwargs): ) -def test_a_mixed_audience_through_create_dataset_writes_both_layouts(pair): +def test_a_mixed_audience_through_create_dataset_writes_both_layouts( + pair, advertised_schemas +): ds_manager, do_manager = pair - _create_for_a_mixed_audience(ds_manager, do_manager, "mixed") + _create_for_a_mixed_audience(advertised_schemas, ds_manager, do_manager, "mixed") public = do_manager._mock_collections_for("mixed") assert {do_manager._protocol_of(c) for c in public} == {"0", "1"} -def test_every_copy_uploads_its_own_private_collection(pair): +def test_every_copy_uploads_its_own_private_collection(pair, advertised_schemas): # Each copy holds its own private directory. An upload of only the newest # leaves the other copies local, and a cold start does not restore them. ds_manager, do_manager = pair - _create_for_a_mixed_audience(ds_manager, do_manager, "mixed", upload_private=True) + _create_for_a_mixed_audience( + advertised_schemas, ds_manager, do_manager, "mixed", upload_private=True + ) private = do_manager._private_collections_for("mixed") assert {do_manager._protocol_of(c) for c in private} == {"0", "1"} -def test_a_cold_start_restores_the_private_data_of_every_copy(pair): +def test_a_cold_start_restores_the_private_data_of_every_copy(pair, advertised_schemas): import shutil ds_manager, do_manager = pair - _create_for_a_mixed_audience(ds_manager, do_manager, "mixed", upload_private=True) + _create_for_a_mixed_audience( + advertised_schemas, ds_manager, do_manager, "mixed", upload_private=True + ) storage = do_manager.dataset_manager.storage private_dirs = { @@ -436,7 +440,9 @@ def _collections_for(do_manager, tag: str): return {do_manager._protocol_of(c) for c in do_manager._mock_collections_for(tag)} -def test_sharing_with_a_protocol0_peer_materializes_the_flat_copy(pair): +def test_sharing_with_a_protocol0_peer_materializes_the_flat_copy( + pair, advertised_schemas +): # A share is a change of audience. The audience decided the layouts at # create time, so a new audience member of another protocol needs a copy # in its layout -- granting it the versioned collection gives it a folder @@ -445,9 +451,7 @@ def test_sharing_with_a_protocol0_peer_materializes_the_flat_copy(pair): _create_for_the_current_audience(ds_manager, do_manager, "afterthought") assert _collections_for(do_manager, "afterthought") == {"1"} - do_manager.peer_manager.live_peer_schemas("syft-dataset")[OLD_PEER] = ( - _dataset_schema("0") - ) + advertised_schemas[OLD_PEER, "syft-dataset"] = _dataset_schema("0") do_manager.share_dataset("afterthought", [OLD_PEER], sync=False) assert _collections_for(do_manager, "afterthought") == {"0", "1"} @@ -458,15 +462,13 @@ def test_sharing_with_a_protocol0_peer_materializes_the_flat_copy(pair): assert flat_dir.exists() -def test_sharing_with_a_current_peer_creates_no_extra_copy(pair): +def test_sharing_with_a_current_peer_creates_no_extra_copy(pair, advertised_schemas): # The control: a peer of our own protocol reads the existing layout, so # the test above measures the fill and not an unconditional copy. ds_manager, do_manager = pair _create_for_the_current_audience(ds_manager, do_manager, "current share") - do_manager.peer_manager.live_peer_schemas("syft-dataset")["new@test.org"] = ( - _dataset_schema("1") - ) + advertised_schemas["new@test.org", "syft-dataset"] = _dataset_schema("1") do_manager.share_dataset("current share", ["new@test.org"], sync=False) assert _collections_for(do_manager, "current share") == {"1"} @@ -483,7 +485,9 @@ def test_sharing_with_an_unknown_peer_materializes_the_widest_layout(pair): assert _collections_for(do_manager, "unknown share") == {"0", "1"} -def test_a_copy_materialized_at_share_time_uploads_its_private_collection(pair): +def test_a_copy_materialized_at_share_time_uploads_its_private_collection( + pair, advertised_schemas +): # Each copy holds its own private directory (see the cold-start test # above). A copy created at share time must follow the same rule, or a # cold start loses its private data. @@ -492,16 +496,14 @@ def test_a_copy_materialized_at_share_time_uploads_its_private_collection(pair): ds_manager, do_manager, "private fill", upload_private=True ) - do_manager.peer_manager.live_peer_schemas("syft-dataset")[OLD_PEER] = ( - _dataset_schema("0") - ) + advertised_schemas[OLD_PEER, "syft-dataset"] = _dataset_schema("0") do_manager.share_dataset("private fill", [OLD_PEER], sync=False) private = do_manager._private_collections_for("private fill") assert {do_manager._protocol_of(c) for c in private} == {"0", "1"} -def test_a_share_uploads_a_local_copy_that_has_no_collection(pair): +def test_a_share_uploads_a_local_copy_that_has_no_collection(pair, advertised_schemas): # A share that fails after the migrate leaves the copy on disk with no # collection of its own. The next share must upload that copy. A second # write of the same layout raises, and the share then grants nothing at @@ -511,9 +513,7 @@ def test_a_share_uploads_a_local_copy_that_has_no_collection(pair): do_manager.dataset_manager.migrate("half done", "0", users=[ds_manager.email]) assert _collections_for(do_manager, "half done") == {"1"} - do_manager.peer_manager.live_peer_schemas("syft-dataset")[OLD_PEER] = ( - _dataset_schema("0") - ) + advertised_schemas[OLD_PEER, "syft-dataset"] = _dataset_schema("0") do_manager.share_dataset("half done", [OLD_PEER], sync=False) assert _collections_for(do_manager, "half done") == {"0", "1"} @@ -560,7 +560,9 @@ def test_upgrade_publishes_the_current_layout_and_the_peer_prefers_it(pair): ) -def test_a_peer_approved_after_an_any_create_gets_a_layout_it_reads(pair): +def test_a_peer_approved_after_an_any_create_gets_a_layout_it_reads( + pair, advertised_schemas +): # An "any" dataset holds the layouts its audience read at create time. A # peer approved later may read none of them, so the approval materializes # and publishes its layout before granting on the collection. @@ -578,15 +580,13 @@ def test_a_peer_approved_after_an_any_create_gets_a_layout_it_reads(pair): assert _collections_for(do_manager, "open dataset") == {"1"} # A protocol-0 peer arrives and is approved. - do_manager.peer_manager.live_peer_schemas("syft-dataset")[OLD_PEER] = ( - _dataset_schema("0") - ) + advertised_schemas[OLD_PEER, "syft-dataset"] = _dataset_schema("0") do_manager._share_any_datasets_with_peer(OLD_PEER) assert _collections_for(do_manager, "open dataset") == {"0", "1"} -def test_a_backfilled_any_layout_is_marked_any_too(pair): +def test_a_backfilled_any_layout_is_marked_any_too(pair, advertised_schemas): # The "any" bit lives on the collection. A layout added at approval time # must carry it, or a later cache rebuild from has_any_permission loses it # and the next peer never sees that layout. @@ -602,9 +602,7 @@ def test_a_backfilled_any_layout_is_marked_any_too(pair): ) do_manager._upload_dataset_to_collection(created["1"], users="any") - do_manager.peer_manager.live_peer_schemas("syft-dataset")[OLD_PEER] = ( - _dataset_schema("0") - ) + advertised_schemas[OLD_PEER, "syft-dataset"] = _dataset_schema("0") do_manager._share_any_datasets_with_peer(OLD_PEER) collections = do_manager._mock_collections_for("open dataset") diff --git a/tests/migrations/p2p/test_dataset_schema_negotiation.py b/tests/migrations/p2p/test_dataset_schema_negotiation.py index d189072e5ae..bf805aa471b 100644 --- a/tests/migrations/p2p/test_dataset_schema_negotiation.py +++ b/tests/migrations/p2p/test_dataset_schema_negotiation.py @@ -26,7 +26,7 @@ def _dataset_schema(protocol_version: str) -> ProtocolSchema: def _storage(tmp_path: Path, peer_schemas: dict) -> DatasetStorage: config = SyftBoxConfig(syftbox_folder=tmp_path / "SyftBox", email=OWNER_EMAIL) (tmp_path / "SyftBox" / OWNER_EMAIL).mkdir(parents=True, exist_ok=True) - return DatasetStorage(config=config, peer_schemas=peer_schemas) + return DatasetStorage(config=config, peer_schema=peer_schemas.get) def test_mixed_audience_writes_both_versions(tmp_path): @@ -63,7 +63,7 @@ def test_no_audience_needs_no_older_layout(tmp_path): assert storage.target_protocol_versions_for_peers() == set() -def test_live_map_updates_are_seen_by_storage(tmp_path): +def test_schema_learned_later_is_seen_by_storage(tmp_path): live: dict = {} storage = _storage(tmp_path, live) assert storage.target_protocol_versions_for_peers([NEW_PEER]) == { @@ -75,12 +75,12 @@ def test_live_map_updates_are_seen_by_storage(tmp_path): } -def test_manager_from_config_passes_schemas_through(tmp_path): +def test_manager_from_config_passes_schema_lookup_through(tmp_path): config = SyftBoxConfig(syftbox_folder=tmp_path / "SyftBox", email=OWNER_EMAIL) (tmp_path / "SyftBox" / OWNER_EMAIL).mkdir(parents=True, exist_ok=True) - live = {OLD_PEER: _dataset_schema("0")} - manager = SyftDatasetManager.from_config(config, peer_schemas=live) - assert manager.storage.peer_schemas is live + lookup = {OLD_PEER: _dataset_schema("0")}.get + manager = SyftDatasetManager.from_config(config, peer_schema=lookup) + assert manager.storage.peer_schema is lookup def test_newer_peer_clamps_to_our_protocol(tmp_path): diff --git a/tests/migrations/p2p/test_job_protocol_skew_delivery.py b/tests/migrations/p2p/test_job_protocol_skew_delivery.py index 96d59fe7345..d94d2104d78 100644 --- a/tests/migrations/p2p/test_job_protocol_skew_delivery.py +++ b/tests/migrations/p2p/test_job_protocol_skew_delivery.py @@ -48,12 +48,10 @@ def _submit(ds_manager, do_manager, job_name: str) -> Path: return project_dir -def test_a_job_for_a_protocol0_peer_uses_the_flat_layout(pair): +def test_a_job_for_a_protocol0_peer_uses_the_flat_layout(pair, advertised_schemas): ds_manager, do_manager = pair # The DO advertises job protocol 0, as a client of 0.1.38 or earlier does. - ds_manager.peer_manager.live_peer_schemas("syft-job")[do_manager.email] = ( - _job_schema("0") - ) + advertised_schemas[do_manager.email, "syft-job"] = _job_schema("0") ref = ds_manager.job_client.manager.new_submission_ref(do_manager.email, "skew.job") assert ref.protocol_version == "0" @@ -62,11 +60,9 @@ def test_a_job_for_a_protocol0_peer_uses_the_flat_layout(pair): ) -def test_a_job_for_a_protocol0_peer_arrives_and_reads(pair): +def test_a_job_for_a_protocol0_peer_arrives_and_reads(pair, advertised_schemas): ds_manager, do_manager = pair - ds_manager.peer_manager.live_peer_schemas("syft-job")[do_manager.email] = ( - _job_schema("0") - ) + advertised_schemas[do_manager.email, "syft-job"] = _job_schema("0") _submit(ds_manager, do_manager, "skew.job") diff --git a/tests/migrations/p2p/test_job_schema_negotiation.py b/tests/migrations/p2p/test_job_schema_negotiation.py index f1051e5f8c7..e87dc8dfc31 100644 --- a/tests/migrations/p2p/test_job_schema_negotiation.py +++ b/tests/migrations/p2p/test_job_schema_negotiation.py @@ -27,7 +27,7 @@ def _storage(tmp_path: Path, peer_schemas: dict) -> JobStorage: syftbox_folder=tmp_path / "SyftBox", current_user_email=DS_EMAIL ) (tmp_path / "SyftBox" / DS_EMAIL).mkdir(parents=True, exist_ok=True) - return JobStorage(config=config, peer_schemas=peer_schemas) + return JobStorage(config=config, peer_schema=peer_schemas.get) def test_protocol0_peer_negotiates_down(tmp_path): @@ -68,14 +68,14 @@ def test_live_map_updates_are_seen_by_storage(tmp_path): assert storage.new_submission_ref(DO_EMAIL, "after.job").protocol_version == "0" -def test_job_client_from_config_passes_schemas_through(tmp_path): +def test_job_client_from_config_passes_schema_lookup_through(tmp_path): config = SyftJobConfig( syftbox_folder=tmp_path / "SyftBox", current_user_email=DS_EMAIL ) (tmp_path / "SyftBox" / DS_EMAIL).mkdir(parents=True, exist_ok=True) - live = {DO_EMAIL: _job_schema("0")} - client = JobClient.from_config(config, peer_schemas=live) - assert client.manager.peer_schemas is live + lookup = {DO_EMAIL: _job_schema("0")}.get + client = JobClient.from_config(config, peer_schema=lookup) + assert client.manager.peer_schema is lookup def test_newer_peer_clamps_to_our_protocol(tmp_path): diff --git a/tests/migrations/p2p/test_message_protocol_downgrade.py b/tests/migrations/p2p/test_message_protocol_downgrade.py index 1b5119cdc6c..864cedfcdfb 100644 --- a/tests/migrations/p2p/test_message_protocol_downgrade.py +++ b/tests/migrations/p2p/test_message_protocol_downgrade.py @@ -112,7 +112,7 @@ def _wire_version(manager, peer_email: str, raw: bytes) -> str: """ peer_store = manager.peer_manager.peer_store opened = peer_store.decrypt_and_verify_if_needed(peer_email, raw) - if peer_store.peer_uses_encryption(peer_email): + if peer_store.pair_encrypts(peer_store.get_cached_peer(peer_email)): assert opened != raw, "encryption is on, but the wire holds plaintext" return json.loads(uncompress_data(opened))["version"] @@ -134,14 +134,12 @@ def _raw_outbox_versions(ds_manager, do_email: str) -> list[str]: def test_a_v2_proposal_for_a_protocol0_peer_downgrades_on_the_wire( - pair, v2_wire_envelopes + pair, v2_wire_envelopes, advertised_schemas ): ds_manager, do_manager = pair v2_proposed, _ = v2_wire_envelopes # The DO advertises client protocol 0, as a client of 0.1.117 or earlier does. - ds_manager.peer_manager.live_peer_schemas("syft")[do_manager.email] = ( - _client_schema("0") - ) + advertised_schemas[do_manager.email, "syft"] = _client_schema("0") message = v2_proposed(**mock_message().model_dump(exclude={"version"})) ds_manager._connection_router.watcher_send_proposed_file_changes_message( @@ -155,13 +153,11 @@ def test_a_v2_proposal_for_a_protocol0_peer_downgrades_on_the_wire( def test_a_v2_events_message_for_a_protocol0_peer_downgrades_on_the_wire( - pair, v2_wire_envelopes + pair, v2_wire_envelopes, advertised_schemas ): ds_manager, do_manager = pair _, v2_events = v2_wire_envelopes - do_manager.peer_manager.live_peer_schemas("syft")[ds_manager.email] = ( - _client_schema("0") - ) + advertised_schemas[ds_manager.email, "syft"] = _client_schema("0") message = v2_events( **get_mock_events_messages(1)[0].model_dump(exclude={"version"}) @@ -173,12 +169,12 @@ def test_a_v2_events_message_for_a_protocol0_peer_downgrades_on_the_wire( assert _raw_outbox_versions(ds_manager, do_manager.email) == ["1"] -def test_a_send_beyond_the_peers_floor_raises(pair): +def test_a_send_beyond_the_peers_floor_raises(pair, advertised_schemas): # A future peer that dropped support for our protocol. Sending anyway would # put up a blob the peer refuses; the negotiation must fail loudly instead. ds_manager, do_manager = pair - ds_manager.peer_manager.live_peer_schemas("syft")[do_manager.email] = ( - _client_schema("2", min_supported_version="2") + advertised_schemas[do_manager.email, "syft"] = _client_schema( + "2", min_supported_version="2" ) with pytest.raises(MigrationError): @@ -187,11 +183,13 @@ def test_a_send_beyond_the_peers_floor_raises(pair): ) -def test_a_send_to_an_unknown_peer_warns_and_keeps_the_current_version(pair, caplog): +def test_a_send_to_an_unknown_peer_warns_and_keeps_the_current_version( + pair, caplog, advertised_schemas +): # Same policy as jobs: a peer without a known schema is assumed to run the # current protocol, and the assumption is logged. ds_manager, do_manager = pair - ds_manager.peer_manager.live_peer_schemas("syft").pop(do_manager.email, None) + advertised_schemas[do_manager.email, "syft"] = None with caplog.at_level(logging.WARNING): ds_manager._connection_router.watcher_send_proposed_file_changes_message( @@ -202,13 +200,11 @@ def test_a_send_to_an_unknown_peer_warns_and_keeps_the_current_version(pair, cap assert _raw_proposal_version(do_manager, ds_manager.email) == "1" -def test_a_current_protocol_peer_gets_the_current_version(pair): +def test_a_current_protocol_peer_gets_the_current_version(pair, advertised_schemas): # The control: a peer on our own protocol gets the current version, so the # tests above measure the downgrade and not a broken default. ds_manager, do_manager = pair - ds_manager.peer_manager.live_peer_schemas("syft")[do_manager.email] = ( - _client_schema("1") - ) + advertised_schemas[do_manager.email, "syft"] = _client_schema("1") ds_manager._connection_router.watcher_send_proposed_file_changes_message( do_manager.email, mock_message() diff --git a/tests/migrations/p2p/test_older_protocol_compatibility.py b/tests/migrations/p2p/test_older_protocol_compatibility.py index 594bebc1fb0..14bf2416a2e 100644 --- a/tests/migrations/p2p/test_older_protocol_compatibility.py +++ b/tests/migrations/p2p/test_older_protocol_compatibility.py @@ -22,7 +22,7 @@ from syft.sync.events.file_change_event import FileChangeEventsMessage from syft.sync.messages.proposed_filechange import ProposedFileChangesMessage from syft.sync.utils.syftbox_utils import uncompress_data -from syft.sync.version.version_info import VersionInfo +from syft.sync.version.peer_info import PeerInfo FIXTURES_DIR = Path(__file__).parent / "fixtures" RELEASE_FIXTURES = sorted(FIXTURES_DIR.glob("*-protocol*")) @@ -56,11 +56,14 @@ def test_version_file_reads_and_round_trips(fixture: Path): raw = _single(fixture, "SYFT_version.json").read_text() assert ("canonical_name" in json.loads(raw)) == _has_identity(_protocol_of(fixture)) - info = VersionInfo.from_json(raw) - assert info.version == client_registry.latest_version("VersionInfo") - assert info.syft_client_version + # Read as a client reads a peer's file: a release before PeerInfo wrote a + # bare VersionInfo. + info = PeerInfo.from_json(raw) + assert info.version == client_registry.latest_version("PeerInfo") + assert info.version_info.version == client_registry.latest_version("VersionInfo") + assert info.version_info.syft_client_version # Round-trip: what the current code writes must load again. - assert VersionInfo.from_json(info.to_json()) == info + assert PeerInfo.from_json(info.to_json()) == info @released_fixtures diff --git a/tests/migrations/p2p/test_private_dataset_protocol_skew.py b/tests/migrations/p2p/test_private_dataset_protocol_skew.py index c725e46117b..7fbcdd057a1 100644 --- a/tests/migrations/p2p/test_private_dataset_protocol_skew.py +++ b/tests/migrations/p2p/test_private_dataset_protocol_skew.py @@ -37,13 +37,13 @@ def pair(): ) -def _create(ds_manager, do_manager, name: str, mixed_audience: bool): +def _create( + advertised_schemas, ds_manager, do_manager, name: str, mixed_audience: bool +): """Create a dataset locally; with a mixed audience both layouts exist.""" users = [ds_manager.email] if mixed_audience: - do_manager.peer_manager.live_peer_schemas("syft-dataset")[OLD_PEER] = ( - _dataset_schema("0") - ) + advertised_schemas[OLD_PEER, "syft-dataset"] = _dataset_schema("0") users.append(OLD_PEER) mock_path, private_path, readme_path = create_tmp_dataset_files() return do_manager.create_dataset( @@ -68,13 +68,15 @@ def _shipped_private_paths(ds_manager, do_manager) -> list[str]: ] -def test_private_files_for_a_protocol0_peer_ship_in_the_flat_layout(pair): +def test_private_files_for_a_protocol0_peer_ship_in_the_flat_layout( + pair, advertised_schemas +): ds_manager, do_manager = pair - _create(ds_manager, do_manager, "mixed private", mixed_audience=True) - # The recipient advertises dataset protocol 0, as an earlier client does. - do_manager.peer_manager.live_peer_schemas("syft-dataset")[ds_manager.email] = ( - _dataset_schema("0") + _create( + advertised_schemas, ds_manager, do_manager, "mixed private", mixed_audience=True ) + # The recipient advertises dataset protocol 0, as an earlier client does. + advertised_schemas[ds_manager.email, "syft-dataset"] = _dataset_schema("0") do_manager.share_private_dataset("mixed private", ds_manager.email) @@ -86,11 +88,15 @@ def test_private_files_for_a_protocol0_peer_ship_in_the_flat_layout(pair): ) -def test_private_files_for_a_current_peer_ship_in_the_versioned_layout(pair): +def test_private_files_for_a_current_peer_ship_in_the_versioned_layout( + pair, advertised_schemas +): # The control: a peer of our own protocol gets the newest layout, so the # test above measures negotiation and not a broken default. ds_manager, do_manager = pair - _create(ds_manager, do_manager, "mixed private", mixed_audience=True) + _create( + advertised_schemas, ds_manager, do_manager, "mixed private", mixed_audience=True + ) do_manager.share_private_dataset("mixed private", ds_manager.email) @@ -100,19 +106,19 @@ def test_private_files_for_a_current_peer_ship_in_the_versioned_layout(pair): assert path.startswith("private/syft_datasets/v1/mixed private/") -def test_a_missing_flat_copy_is_materialized_for_a_protocol0_peer(pair): +def test_a_missing_flat_copy_is_materialized_for_a_protocol0_peer( + pair, advertised_schemas +): # The dataset was created for a current audience, so no flat copy exists. # The share must create one -- the same fill that share_dataset applies -- # because shipping the v1 paths gives the peer files it never scans. ds_manager, do_manager = pair - _create(ds_manager, do_manager, "v1 only", mixed_audience=False) + _create(advertised_schemas, ds_manager, do_manager, "v1 only", mixed_audience=False) storage = do_manager.dataset_manager.storage flat_dir = storage.private_dataset_dir(storage.new_dataset_ref("v1 only", "0")) assert not flat_dir.exists(), "the flat copy should not exist before the share" - do_manager.peer_manager.live_peer_schemas("syft-dataset")[ds_manager.email] = ( - _dataset_schema("0") - ) + advertised_schemas[ds_manager.email, "syft-dataset"] = _dataset_schema("0") do_manager.share_private_dataset("v1 only", ds_manager.email) paths = _shipped_private_paths(ds_manager, do_manager) diff --git a/tests/migrations/p2p/test_unknown_peer_forced_path.py b/tests/migrations/p2p/test_unknown_peer_forced_path.py index f680d87ce70..238d435ea8e 100644 --- a/tests/migrations/p2p/test_unknown_peer_forced_path.py +++ b/tests/migrations/p2p/test_unknown_peer_forced_path.py @@ -27,13 +27,13 @@ def _job_storage(tmp_path: Path) -> JobStorage: syftbox_folder=tmp_path / "SyftBox", current_user_email=DS_EMAIL ) (tmp_path / "SyftBox" / DS_EMAIL).mkdir(parents=True, exist_ok=True) - return JobStorage(config=config, peer_schemas={}) + return JobStorage(config=config) def _dataset_storage(tmp_path: Path) -> DatasetStorage: config = SyftBoxConfig(syftbox_folder=tmp_path / "SyftBox", email=DO_EMAIL) (tmp_path / "SyftBox" / DO_EMAIL).mkdir(parents=True, exist_ok=True) - return DatasetStorage(config=config, peer_schemas={}) + return DatasetStorage(config=config) def test_a_forced_job_reports_the_assumed_protocol(tmp_path, caplog): @@ -65,11 +65,13 @@ def test_a_known_peer_reports_nothing(tmp_path, caplog): from syft_migration import ProtocolSchema storage = _job_storage(tmp_path) - storage.peer_schemas[DO_EMAIL] = ProtocolSchema( - protocol_name="syft-job", - version=JOB_PROTOCOL_VERSION, - supported_versions={"JobState": ["1"]}, - ) + storage.peer_schema = { + DO_EMAIL: ProtocolSchema( + protocol_name="syft-job", + version=JOB_PROTOCOL_VERSION, + supported_versions={"JobState": ["1"]}, + ) + }.get with caplog.at_level(logging.WARNING): storage.negotiated_protocol_version_for_peer(DO_EMAIL) assert caplog.records == [] diff --git a/tests/migrations/unit/test_objects_registered.py b/tests/migrations/unit/test_objects_registered.py index 7114fb0d883..9f1b5cc993d 100644 --- a/tests/migrations/unit/test_objects_registered.py +++ b/tests/migrations/unit/test_objects_registered.py @@ -13,7 +13,12 @@ from syft.sync.version.version_info import VersionInfo, VersionInfoV2 from syft_migration import unregistered_objects, versioned_objects -OBJECTS = {"VersionInfo", "FileChangeEventsMessage", "ProposedFileChangesMessage"} +OBJECTS = { + "VersionInfo", + "PeerInfo", + "FileChangeEventsMessage", + "ProposedFileChangesMessage", +} def test_versioned_objects_registered_and_aliased(): diff --git a/tests/migrations/unit/test_peer_info_fields.py b/tests/migrations/unit/test_peer_info_fields.py new file mode 100644 index 00000000000..86b218f79d2 --- /dev/null +++ b/tests/migrations/unit/test_peer_info_fields.py @@ -0,0 +1,32 @@ +"""PeerInfo is the published version file, so it follows the same rules as +VersionInfo (see test_version_info_fields): a field of an older version must not +disappear or change name, and a field that a newer version adds needs a default. +""" + +import syft # noqa: F401 -- imports models and registers history +from syft.sync.encryption_policy import EncryptionPolicy +from syft.sync.version.peer_info import PeerInfo, PeerInfoV1 +from syft.sync.version.version_info import VersionInfoV2 + +# Frozen on purpose. A change here means a change to the published version file. +V1_FIELDS = {"canonical_name", "version", "version_info", "encryption_policy"} + + +def test_v1_fields_are_frozen(): + assert set(PeerInfoV1.model_fields) == V1_FIELDS, ( + "PeerInfoV1 changed. Peers read this object, so a removed or renamed " + "field stops them from parsing the version file of this client." + ) + + +def test_version_file_from_before_peer_info_reads_as_optional(): + written_by_an_older_client = VersionInfoV2( + syft_client_version="0.10.1", + min_supported_syft_client_version="0.10.0", + protocol_version="1.0.0", + min_supported_protocol_version="1.0.0", + ).to_json() + + info = PeerInfo.from_json(written_by_an_older_client) + assert info.version_info.syft_client_version == "0.10.1" + assert info.encryption_policy == EncryptionPolicy.OPTIONAL diff --git a/tests/migrations/unit/test_peer_manager_schemas.py b/tests/migrations/unit/test_peer_manager_schemas.py index cab3d0e52c9..f4452d7b206 100644 --- a/tests/migrations/unit/test_peer_manager_schemas.py +++ b/tests/migrations/unit/test_peer_manager_schemas.py @@ -1,17 +1,23 @@ -"""PeerManager's live peer-schema maps track loaded peer versions.""" +"""PeerStore.protocol_schema reads the schemas a peer advertises in its peer info.""" -from syft.sync.version.peer_manager import PeerManager +from syft.sync.encryption_policy import EncryptionPolicy +from syft.sync.peers.peer import Peer +from syft.sync.peers.peer_store import PeerStore +from syft.sync.version.peer_info import PeerInfo from syft.sync.version.version_info import VersionInfo, VersionInfoV1 +PEER = "do@test.org" -def _peer_manager() -> PeerManager: - # Construct without model_validate side effects: only the private schema - # map and _update_peer_schemas are exercised here. - return PeerManager.model_construct() - -def _v2_with_schemas() -> VersionInfo: - return VersionInfo.current() +def _store_with_peer(version_info) -> PeerStore: + store = PeerStore(email="ds@test.org") + info = None + if version_info is not None: + info = PeerInfo.model_construct( + version_info=version_info, encryption_policy=EncryptionPolicy.OPTIONAL + ) + store.add_peer(Peer(email=PEER, info=info)) + return store def _v1() -> VersionInfoV1: @@ -23,42 +29,25 @@ def _v1() -> VersionInfoV1: ) -def test_advertising_peer_appears_in_live_map(): - pm = _peer_manager() - live = pm.live_peer_schemas("syft-job") - pm._update_peer_schemas("do@test.org", _v2_with_schemas()) - assert "do@test.org" in live - assert live["do@test.org"].protocol_name == "syft-job" +def test_advertising_peer_has_schema(): + store = _store_with_peer(VersionInfo.current()) + assert store.protocol_schema(PEER, "syft-job").protocol_name == "syft-job" def test_pre_v2_peer_is_an_unknown_speaker(): - pm = _peer_manager() - live = pm.live_peer_schemas("syft-job") - pm._update_peer_schemas("do@test.org", _v2_with_schemas()) - # A reloaded version file from an old client (upgraded V1: no schemas) - # must remove the stale entry. - pm._update_peer_schemas("do@test.org", _v1()) - assert live == {} - - -def test_cleared_version_removes_peer(): - pm = _peer_manager() - live = pm.live_peer_schemas("syft") - pm._update_peer_schemas("do@test.org", _v2_with_schemas()) - pm._update_peer_schemas("do@test.org", None) - assert live == {} - - -def test_map_identity_is_stable(): - # live_peer_schemas must always return the same dict object so consumers - # holding a reference see updates. - pm = _peer_manager() - assert pm.live_peer_schemas("syft-job") is pm.live_peer_schemas("syft-job") - - -def test_late_registered_map_backfills_from_loaded_versions(): - pm = _peer_manager() - pm._update_peer_schemas("do@test.org", _v2_with_schemas()) - # Registering the protocol AFTER the version loaded must not start empty. - live = pm.live_peer_schemas("syft-job") - assert "do@test.org" in live + # A version file from an old client (V1) advertises no schemas. + assert _store_with_peer(_v1()).protocol_schema(PEER, "syft-job") is None + + +def test_peer_without_info_has_no_schema(): + assert _store_with_peer(None).protocol_schema(PEER, "syft") is None + + +def test_peer_not_in_store_has_no_schema(): + assert PeerStore(email="ds@test.org").protocol_schema(PEER, "syft") is None + + +def test_schema_follows_reloaded_info(): + store = _store_with_peer(VersionInfo.current()) + store.get_cached_peer(PEER).info = None + assert store.protocol_schema(PEER, "syft-job") is None diff --git a/tests/unit/syft_bg/test_init.py b/tests/unit/syft_bg/test_init.py index 996742f1dfc..03a1e44e5fe 100644 --- a/tests/unit/syft_bg/test_init.py +++ b/tests/unit/syft_bg/test_init.py @@ -5,7 +5,17 @@ from pathlib import Path from unittest.mock import patch +import pytest + +from syft_rds import ( + EncryptionPolicy, + EncryptionPolicyMismatchError, + datasite_settings_path, + resolve_encryption_policy, +) + from syft_bg.api.api import init +from syft_bg.cli.init import InitFlowError, UserPassedConfig, run_init_flow from syft_bg.api.utils import move_token_to_syftbg_dir from syft_bg.common.config import get_default_paths from syft_bg.common.syft_bg_config import SyftBgConfig @@ -60,10 +70,83 @@ def test_sets_both_gmail_and_drive_token_path(self, temp_dir): token_source.write_text("{}") with _patched_paths(temp_dir) as patched: - init(do_email="alice@test.com", token_path=token_source) + init( + do_email="alice@test.com", + syftbox_root=temp_dir / "SyftBox", + token_path=token_source, + ) config = SyftBgConfig.from_path(patched.config) canonical_token_path = temp_dir / "syftbg" / "token.json" assert config.drive_token_path == canonical_token_path assert config.gmail_token_path == canonical_token_path + + +def _policy_on_disk(syftbox_root: Path) -> EncryptionPolicy: + path = datasite_settings_path(syftbox_root, "alice@test.com") + return resolve_encryption_policy(path, None) + + +class TestInitEncryption: + """api.init(encryption=...) sets the policy of the SyftBox, not of syft-bg.""" + + def test_new_syftbox_without_policy_is_optional(self, temp_dir): + with _patched_paths(temp_dir): + init(do_email="alice@test.com", syftbox_root=temp_dir / "SyftBox") + assert _policy_on_disk(temp_dir / "SyftBox") == EncryptionPolicy.OPTIONAL + + def test_required_is_written_to_syftbox(self, temp_dir): + with _patched_paths(temp_dir) as patched: + init( + do_email="alice@test.com", + syftbox_root=temp_dir / "SyftBox", + encryption=True, + ) + config = SyftBgConfig.from_path(patched.config) + assert _policy_on_disk(temp_dir / "SyftBox") == EncryptionPolicy.REQUIRED + assert "encryption_policy" not in config.model_dump() + + def test_other_policy_than_syftbox_raises(self, temp_dir): + with _patched_paths(temp_dir): + init(do_email="alice@test.com", syftbox_root=temp_dir / "SyftBox") + with pytest.raises(EncryptionPolicyMismatchError): + init( + do_email="alice@test.com", + syftbox_root=temp_dir / "SyftBox", + encryption=True, + ) + + +class TestInitFlowEncryption: + """The init flow sets the policy of a new SyftBox and keeps that of an old one.""" + + def _run_quiet(self, temp_dir, **kwargs): + with _patched_paths(temp_dir) as patched: + with patch("syft_bg.cli.init.flow.get_default_paths", return_value=patched): + run_init_flow( + UserPassedConfig( + email="alice@test.com", + syftbox_root=str(temp_dir / "SyftBox"), + quiet=True, + **kwargs, + ) + ) + + def test_quiet_mode_without_policy_is_optional(self, temp_dir): + self._run_quiet(temp_dir) + assert _policy_on_disk(temp_dir / "SyftBox") == EncryptionPolicy.OPTIONAL + + def test_quiet_mode_sets_policy(self, temp_dir): + self._run_quiet(temp_dir, encryption="required") + assert _policy_on_disk(temp_dir / "SyftBox") == EncryptionPolicy.REQUIRED + + def test_quiet_mode_without_policy_keeps_syftbox_policy(self, temp_dir): + self._run_quiet(temp_dir, encryption="required") + self._run_quiet(temp_dir) + assert _policy_on_disk(temp_dir / "SyftBox") == EncryptionPolicy.REQUIRED + + def test_other_policy_than_syftbox_raises(self, temp_dir): + self._run_quiet(temp_dir, encryption="optional") + with pytest.raises(InitFlowError, match="encryption policy"): + self._run_quiet(temp_dir, encryption="required") diff --git a/tests/unit/syft_bg/test_orchestrator_encryption.py b/tests/unit/syft_bg/test_orchestrator_encryption.py new file mode 100644 index 00000000000..de5fbdb38d0 --- /dev/null +++ b/tests/unit/syft_bg/test_orchestrator_encryption.py @@ -0,0 +1,53 @@ +"""The syft-bg orchestrators build their client with the policy of the SyftBox.""" + +from unittest.mock import MagicMock + +import pytest +import syft_rds + +from syft_bg.approve.config import AutoApproveConfig +from syft_bg.approve.orchestrator import ApprovalOrchestrator +from syft_bg.sync.config import SyncConfig +from syft_bg.sync.orchestrator import SyncOrchestrator + + +def _sync(temp_dir): + config = SyncConfig( + do_email="do@test.com", + syftbox_root=temp_dir, + sync_state_path=temp_dir / "sync_state.json", + ) + return SyncOrchestrator, config + + +def _approve(temp_dir): + config = AutoApproveConfig(do_email="do@test.com", syftbox_root=temp_dir) + return ApprovalOrchestrator, config + + +@pytest.fixture +def client_kwargs(monkeypatch): + """Capture the keyword arguments the orchestrator builds its client with.""" + seen = {} + monkeypatch.setattr(syft_rds, "check_env", lambda: syft_rds.Environment.JUPYTER) + monkeypatch.setattr( + syft_rds.SyftRDSClientConfig, + "for_jupyter", + classmethod(lambda cls, **kwargs: seen.update(kwargs)), + ) + monkeypatch.setattr( + syft_rds.SyftRDSClient, + "from_config", + classmethod(lambda cls, config: MagicMock()), + ) + monkeypatch.setattr( + ApprovalOrchestrator, "_wait_for_sync_ready", staticmethod(lambda **kw: None) + ) + return seen + + +@pytest.mark.parametrize("build", [_sync, _approve], ids=["sync", "approve"]) +def test_orchestrator_takes_policy_of_syftbox(temp_dir, client_kwargs, build): + orchestrator, config = build(temp_dir) + orchestrator.from_config(config) + assert client_kwargs.get("encryption") is None diff --git a/tests/unit/test_crypto_keys_version.py b/tests/unit/test_crypto_keys_version.py index 785f5b4309d..218445e19da 100644 --- a/tests/unit/test_crypto_keys_version.py +++ b/tests/unit/test_crypto_keys_version.py @@ -1,8 +1,8 @@ -"""The crypto key file carries a version, and an unknown one stops the load. +"""The crypto key file carries a version, and any other version stops the load. A user cannot rebuild a private key, so delete-and-rebuild is not a recovery -here. If a newer client wrote the file, this client must refuse it rather than -read it wrong and lose the keys. +here. A client must refuse a file of another version rather than read it wrong +and lose the keys. """ import json @@ -10,15 +10,21 @@ import pytest from syft.sync.peers.peer_store import CRYPTO_KEYS_VERSION, PeerStore +EMAIL = "alice@example.com" + def _saved(tmp_path): - store = PeerStore(email="alice@example.com", use_encryption=True) - store.generate_keys() path = tmp_path / "crypto_keys.json" - store.save_keys(path) + PeerStore.create(email=EMAIL, keys_path=path) return path +def _set_version(path, version): + data = json.loads(path.read_text()) + data["version"] = version + path.write_text(json.dumps(data)) + + def test_a_saved_file_carries_the_version(tmp_path): data = json.loads(_saved(tmp_path).read_text()) assert data["version"] == CRYPTO_KEYS_VERSION @@ -26,26 +32,54 @@ def test_a_saved_file_carries_the_version(tmp_path): def test_a_saved_file_loads_back(tmp_path): path = _saved(tmp_path) - loaded = PeerStore.load_keys(path) - assert loaded.email == "alice@example.com" + on_disk = path.read_text() + PeerStore.create(email=EMAIL, keys_path=path) + assert path.read_text() == on_disk -def test_a_file_without_a_version_still_loads(tmp_path): - # Written before the version field existed. Those keys must keep working. +@pytest.mark.parametrize( + "version", + [CRYPTO_KEYS_VERSION - 1, CRYPTO_KEYS_VERSION + 1], + ids=["older", "newer"], +) +def test_a_file_of_another_version_is_refused(tmp_path, version): path = _saved(tmp_path) - data = json.loads(path.read_text()) - del data["version"] - path.write_text(json.dumps(data)) + _set_version(path, version) - loaded = PeerStore.load_keys(path) - assert loaded.email == "alice@example.com" + with pytest.raises(ValueError, match=f"version {version}"): + PeerStore.create(email=EMAIL, keys_path=path) -def test_a_file_from_a_newer_client_is_refused(tmp_path): +@pytest.mark.parametrize( + "change", + [lambda d: d.pop("keys_jwk"), lambda d: d.update(peer_bundles={})], + ids=["missing field", "extra field"], +) +def test_a_file_of_another_shape_is_refused(tmp_path, change): path = _saved(tmp_path) data = json.loads(path.read_text()) - data["version"] = CRYPTO_KEYS_VERSION + 1 + change(data) path.write_text(json.dumps(data)) - with pytest.raises(ValueError, match=str(CRYPTO_KEYS_VERSION + 1)): - PeerStore.load_keys(path) + with pytest.raises(ValueError, match="_KeyFile"): + PeerStore.create(email=EMAIL, keys_path=path) + + +def test_a_file_of_another_identity_is_refused(tmp_path): + path = _saved(tmp_path) + + with pytest.raises(ValueError, match="belongs to"): + PeerStore.create(email="bob@example.com", keys_path=path) + + +def test_saving_never_replaces_an_existing_file(tmp_path): + """A second process that generated keys at the same time must not win.""" + path = _saved(tmp_path) + on_disk = path.read_text() + other = PeerStore(email=EMAIL) + other.generate_keys() + + with pytest.raises(FileExistsError): + other.save_keys(path) + assert path.read_text() == on_disk + assert [p.name for p in tmp_path.iterdir()] == ["crypto_keys.json"] diff --git a/tests/unit/test_drive_connections.py b/tests/unit/test_drive_connections.py index 8073ca15b7c..eabfbd7135c 100644 --- a/tests/unit/test_drive_connections.py +++ b/tests/unit/test_drive_connections.py @@ -14,6 +14,7 @@ from syft.sync.connections.connection_router import ConnectionRouter from syft.sync.connections.drive.gdrive_transport import CollectionFolder from syft.sync.connections.drive.mock_drive_service import pair_with_mock_service +from syft.sync.encryption_policy import EncryptionPolicy from syft.sync.events.file_change_event import FileChangeEventsMessage from syft.sync.messages.proposed_filechange import ProposedFileChangesMessage from syft.sync.peers.peer_store import PeerStore @@ -30,7 +31,9 @@ Transport, WatcherStore, ) +from syft.sync.version.peer_info import PeerInfo from syft.sync.version.version_info import VersionInfo +from tests.unit.test_sync_manager import _store_knowing DO = "do@example.com" DS = "ds@example.com" @@ -225,6 +228,12 @@ def test_peering_keeps_the_peer_registry(pair): assert peering.read_peer_registry(force_download=True)[DS]["state"] == "accepted" +def test_reading_own_key_bundle_creates_no_folder(pair): + owner = pair[0].as_transport() + assert owner.peering.read_own_key_bundle(DS) is None + assert pair[0]._find_encryption_bundles_folder_id() is None + + def test_peering_publishes_a_key_bundle_both_sides_can_read(pair): do_conn, ds_conn = pair owner, scientist = do_conn.as_transport(), ds_conn.as_transport() @@ -238,16 +247,19 @@ def test_peering_publishes_a_key_bundle_both_sides_can_read(pair): assert owner.peering.read_own_key_bundle("someone-else@example.com") is None -def test_peering_publishes_a_version_the_peer_can_read(pair): +def test_peering_publishes_peer_info_the_peer_can_read(pair): do_conn, ds_conn = pair owner, scientist = do_conn.as_transport(), ds_conn.as_transport() - version = VersionInfo.current() + info = PeerInfo( + version_info=VersionInfo.current(), + encryption_policy=EncryptionPolicy.REQUIRED, + ) - owner.peering.publish_version(version) - owner.peering.grant_version_read(DS) + owner.peering.publish_peer_info(info) + owner.peering.grant_peer_info_read(DS, info) - assert owner.peering.read_own_version() == version - assert scientist.peering.read_peer_version(DO) == version + assert owner.peering.read_own_peer_info() == info + assert scientist.peering.read_peer_info(DO) == info # -- ConnectionRouter ------------------------------------------------------------- @@ -255,7 +267,7 @@ def test_peering_publishes_a_version_the_peer_can_read(pair): def _router_with_mock_p2p(): do_conn, _ = pair_with_mock_service(DO, DS) - router = ConnectionRouter(connections=[do_conn], peer_store=PeerStore(email=DO)) + router = ConnectionRouter(connections=[do_conn], peer_store=_store_knowing(DO, DS)) drive = do_conn.as_transport() p2p = Mock(spec=P2PConnection) router._transport = drive.model_copy(update={"p2p": p2p}) diff --git a/tests/unit/test_encryption.py b/tests/unit/test_encryption.py index 81fc9dbaeb1..21db1ae4f89 100644 --- a/tests/unit/test_encryption.py +++ b/tests/unit/test_encryption.py @@ -1,13 +1,14 @@ import copy +import json import pickle -import tempfile from pathlib import Path import pytest +from syft.sync.encryption_policy import EncryptionPolicy from syft.sync.peers.exceptions import SyftPeerNotReadyError from syft.sync.peers.peer import Peer, PeerState -from syft.sync.peers.peer_store import PeerStore +from syft.sync.peers.peer_store import PeerStore, import_key_file from syft.sync.syftbox_manager import SyftboxManager from tests.unit.test_sync_manager import path_for_job from tests.unit.utils import grant_job_inbox_access @@ -19,18 +20,22 @@ def test_key_generation(): - ps = PeerStore(email="alice@example.com", use_encryption=True) - assert not ps.has_my_keys() + ps = PeerStore( + email="alice@example.com", encryption_policy=EncryptionPolicy.REQUIRED + ) ps.generate_keys() - assert ps.has_my_keys() assert ps.public_key is not None assert ps.get_public_bundle() is not None def test_encrypt_decrypt_roundtrip(): - alice = PeerStore(email="alice@example.com", use_encryption=True) + alice = PeerStore( + email="alice@example.com", encryption_policy=EncryptionPolicy.REQUIRED + ) alice.generate_keys() - bob = PeerStore(email="bob@example.com", use_encryption=True) + bob = PeerStore( + email="bob@example.com", encryption_policy=EncryptionPolicy.REQUIRED + ) bob.generate_keys() # Add peers so bundles can be stored on Peer objects @@ -50,7 +55,9 @@ def test_encrypt_decrypt_roundtrip(): def test_encrypt_without_keys_raises(): - ps = PeerStore(email="alice@example.com", use_encryption=True) + ps = PeerStore( + email="alice@example.com", encryption_policy=EncryptionPolicy.REQUIRED + ) ps.add_peer(Peer(email="bob@example.com")) with pytest.raises(ValueError, match="Keys are created at login"): ps.encrypt("bob@example.com", b"data") @@ -87,7 +94,9 @@ def test_encrypt_without_peer_bundle_names_the_cause(state, cause, remedy): Naming neither the cause nor the remedy is what left data owners stuck on this error with nothing to act on. """ - ps = PeerStore(email="alice@example.com", use_encryption=True) + ps = PeerStore( + email="alice@example.com", encryption_policy=EncryptionPolicy.REQUIRED + ) ps.generate_keys() ps.add_peer(Peer(email="bob@example.com", state=state)) @@ -116,14 +125,18 @@ def test_peer_not_ready_error_survives_pickle_and_copy(): def test_try_decrypt_no_keys(): - ps = PeerStore(email="alice@example.com", use_encryption=True) + ps = PeerStore( + email="alice@example.com", encryption_policy=EncryptionPolicy.REQUIRED + ) data = b"some unencrypted data" with pytest.raises(ValueError, match="Keys are created at login"): ps.decrypt("bob@example.com", data) == data def test_try_decrypt_no_peer_bundle(): - ps = PeerStore(email="alice@example.com", use_encryption=True) + ps = PeerStore( + email="alice@example.com", encryption_policy=EncryptionPolicy.REQUIRED + ) ps.generate_keys() data = b"some unencrypted data" with pytest.raises(SyftPeerNotReadyError) as exc: @@ -133,9 +146,13 @@ def test_try_decrypt_no_peer_bundle(): def test_try_decrypt_invalid_envelope(): - alice = PeerStore(email="alice@example.com", use_encryption=True) + alice = PeerStore( + email="alice@example.com", encryption_policy=EncryptionPolicy.REQUIRED + ) alice.generate_keys() - bob = PeerStore(email="bob@example.com", use_encryption=True) + bob = PeerStore( + email="bob@example.com", encryption_policy=EncryptionPolicy.REQUIRED + ) bob.generate_keys() alice.add_peer(Peer(email="bob@example.com")) @@ -149,48 +166,61 @@ def test_try_decrypt_invalid_envelope(): bob.decrypt("alice@example.com", bad_data) -def test_save_and_load_keys(): - alice = PeerStore(email="alice@example.com", use_encryption=True) - alice.generate_keys() - bob = PeerStore(email="bob@example.com", use_encryption=True) +def test_key_file_holds_identity_only(tmp_path): + keys_path = tmp_path / "crypto_keys.json" + alice = PeerStore.create(email="alice@example.com", keys_path=keys_path) + bob = PeerStore(email="bob@example.com") bob.generate_keys() - alice.add_peer(Peer(email="bob@example.com")) alice.set_peer_bundle("bob@example.com", bob.get_public_bundle()) - with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f: - path = Path(f.name) + assert set(json.loads(keys_path.read_text())) == {"version", "email", "keys_jwk"} + reloaded = PeerStore.create(email="alice@example.com", keys_path=keys_path) + assert reloaded.get_public_bundle() == alice.get_public_bundle() + assert not reloaded.has_peer_bundle("bob@example.com") + + +def _session_key_file(tmp_path, email="alice@example.com"): + path = tmp_path / "upload" / "crypto_keys.json" + return path, PeerStore.create(email=email, keys_path=path) + + +def test_imported_key_file_restores_keys(tmp_path): + uploaded, earlier = _session_key_file(tmp_path) + keys_path = tmp_path / "SyftBox" / "crypto_keys.json" + + import_key_file(uploaded, keys_path, "alice@example.com") - alice.save_keys(path) - loaded = PeerStore.load_keys(path) + restored = PeerStore.create(email="alice@example.com", keys_path=keys_path) + assert restored.get_public_bundle() == earlier.get_public_bundle() - assert loaded.email == "alice@example.com" - assert loaded.has_my_keys() - assert loaded.get_public_bundle() is not None - assert loaded.has_peer_bundle("bob@example.com") - path.unlink() +def test_import_same_key_file_twice(tmp_path): + uploaded, _ = _session_key_file(tmp_path) + keys_path = tmp_path / "SyftBox" / "crypto_keys.json" + import_key_file(uploaded, keys_path, "alice@example.com") -def test_peer_use_encryption_inherited(): - """Peers added to PeerStore inherit its use_encryption flag.""" - ps = PeerStore(email="alice@example.com", use_encryption=True) - peer = Peer(email="bob@example.com") - assert not peer.use_encryption - ps.add_peer(peer) - assert peer.use_encryption + import_key_file(uploaded, keys_path, "alice@example.com") - ps2 = PeerStore(email="alice@example.com", use_encryption=False) - peer2 = Peer(email="bob@example.com", use_encryption=True) - ps2.add_peer(peer2) - assert not peer2.use_encryption +def test_import_key_file_of_another_identity_refused(tmp_path): + uploaded, _ = _session_key_file(tmp_path, email="bob@example.com") + keys_path = tmp_path / "SyftBox" / "crypto_keys.json" -def test_set_peers_inherits_encryption(): - """set_peers propagates use_encryption to all peers.""" - ps = PeerStore(email="alice@example.com", use_encryption=True) - peers = [Peer(email="bob@example.com"), Peer(email="carol@example.com")] - ps.set_peers(peers) - assert all(p.use_encryption for p in peers) + with pytest.raises(ValueError, match="belongs to"): + import_key_file(uploaded, keys_path, "alice@example.com") + assert not keys_path.exists() + + +def test_import_never_replaces_other_keys(tmp_path): + uploaded, _ = _session_key_file(tmp_path) + keys_path = tmp_path / "SyftBox" / "crypto_keys.json" + PeerStore.create(email="alice@example.com", keys_path=keys_path) + on_disk = keys_path.read_text() + + with pytest.raises(FileExistsError): + import_key_file(uploaded, keys_path, "alice@example.com") + assert keys_path.read_text() == on_disk # ========================================================================= @@ -270,9 +300,6 @@ def test_no_encryption_backward_compat(): grant_job_inbox_access(do_manager, ds_manager.email) - assert ds_manager._peer_store is None - assert do_manager._peer_store is None - file_path = path_for_job(do_manager.email, ds_manager.email, "my.job") ds_manager._send_file_change(file_path, "hello unencrypted") do_manager.sync() @@ -317,7 +344,9 @@ def test_bundle_exchange_through_peer_approval(): def test_self_encrypt_decrypt_roundtrip(): """PeerStore can encrypt and decrypt data for itself.""" - ps = PeerStore(email="alice@example.com", use_encryption=True) + ps = PeerStore( + email="alice@example.com", encryption_policy=EncryptionPolicy.REQUIRED + ) ps.generate_keys() plaintext = b"secret at-rest data" @@ -330,7 +359,9 @@ def test_self_encrypt_decrypt_roundtrip(): def test_self_encrypt_if_needed_with_encryption(): """encrypt_for_self_if_needed encrypts when enabled and keys exist.""" - ps = PeerStore(email="alice@example.com", use_encryption=True) + ps = PeerStore( + email="alice@example.com", encryption_policy=EncryptionPolicy.REQUIRED + ) ps.generate_keys() data = b"hello" @@ -343,7 +374,9 @@ def test_self_encrypt_if_needed_with_encryption(): def test_self_encrypt_if_needed_without_encryption(): """encrypt_for_self_if_needed is a no-op when encryption is disabled.""" - ps = PeerStore(email="alice@example.com", use_encryption=False) + ps = PeerStore( + email="alice@example.com", encryption_policy=EncryptionPolicy.OPTIONAL + ) ps.generate_keys() data = b"hello" @@ -351,13 +384,14 @@ def test_self_encrypt_if_needed_without_encryption(): assert ps.decrypt_and_verify_for_self_if_needed(data) == data -def test_self_encrypt_if_needed_without_keys(): - """encrypt_for_self_if_needed is a no-op when keys are missing.""" - ps = PeerStore(email="alice@example.com", use_encryption=True) +def test_required_self_decrypt_refuses_plaintext(): + ps = PeerStore( + email="alice@example.com", encryption_policy=EncryptionPolicy.REQUIRED + ) + ps.generate_keys() - data = b"hello" - assert ps.encrypt_for_self_if_needed(data) == data - assert ps.decrypt_and_verify_for_self_if_needed(data) == data + with pytest.raises(ValueError, match="SYC envelope"): + ps.decrypt_and_verify_for_self_if_needed(b"hello") # ========================================================================= @@ -417,7 +451,6 @@ def test_encrypted_rolling_state_roundtrip(): FileChangeEvent, FileChangeEventsMessage, ) - from pathlib import Path from uuid import uuid4 ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( @@ -477,9 +510,13 @@ def test_at_rest_no_encryption_backward_compat(): def _setup_alice_bob(): """Helper: create alice and bob PeerStores with exchanged bundles.""" - alice = PeerStore(email="alice@example.com", use_encryption=True) + alice = PeerStore( + email="alice@example.com", encryption_policy=EncryptionPolicy.REQUIRED + ) alice.generate_keys() - bob = PeerStore(email="bob@example.com", use_encryption=True) + bob = PeerStore( + email="bob@example.com", encryption_policy=EncryptionPolicy.REQUIRED + ) bob.generate_keys() alice.add_peer(Peer(email="bob@example.com")) @@ -505,7 +542,9 @@ def test_verify_message_wrong_sender_raises(): """Verification fails when checked against the wrong sender's key.""" alice, bob = _setup_alice_bob() - carol = PeerStore(email="carol@example.com", use_encryption=True) + carol = PeerStore( + email="carol@example.com", encryption_policy=EncryptionPolicy.REQUIRED + ) carol.generate_keys() bob.add_peer(Peer(email="carol@example.com")) bob.set_peer_bundle("carol@example.com", carol.get_public_bundle()) @@ -531,7 +570,9 @@ def test_decrypt_tampered_envelope_raises(): def test_verify_message_from_self(): """Verify signature of a self-encrypted envelope.""" - ps = PeerStore(email="alice@example.com", use_encryption=True) + ps = PeerStore( + email="alice@example.com", encryption_policy=EncryptionPolicy.REQUIRED + ) ps.generate_keys() plaintext = b"self-signed data" diff --git a/tests/unit/test_encryption_policy.py b/tests/unit/test_encryption_policy.py new file mode 100644 index 00000000000..56cd8736859 --- /dev/null +++ b/tests/unit/test_encryption_policy.py @@ -0,0 +1,597 @@ +"""Per-peer encryption negotiated from each side's advertised policy. + +A client is ``optional`` (encrypt only when the peer requires it) or +``required``. A pair encrypts when either side requires it. +""" + +import json + +import pytest +from pydantic import ValidationError + +from syft.sync.connections.connection_router import ConnectionRouter +from syft.sync.connections.drive import mock_drive_service +from syft.sync.encryption_policy import ( + EncryptionPolicy, + EncryptionPolicyMismatchError, + MessageFormatError, +) +from syft.sync.events.file_change_event import ( + FileChangeEventsMessage, +) +from syft.sync.peers.exceptions import SyftPeerNotReadyError +from syft.sync.peers.peer import Peer +from syft.sync.peers.peer_store import ( + PeerStore, + datasite_settings_path, + resolve_encryption_policy, +) +from syft.sync.syftbox_manager import SyftboxManager +from syft.sync.version.peer_info import PeerInfo +from syft.sync.version.peer_manager import PeerManager, PeerManagerConfig +from syft.sync.version.version_info import VersionInfo +from tests.unit.test_persisted_cache import ( + _assert_ds_receives_after_notebook_sync, + _pair_with_unsent_do_file, +) +from tests.unit.test_sync_manager import path_for_job +from tests.unit.utils import grant_job_inbox_access + +OPTIONAL = EncryptionPolicy.OPTIONAL +REQUIRED = EncryptionPolicy.REQUIRED + + +def _peer_info(policy: EncryptionPolicy | None) -> PeerInfo | None: + """The info a peer publishes, or None when it was not read yet.""" + if policy is None: + return None + return PeerInfo(version_info=VersionInfo.current(), encryption_policy=policy) + + +def _store_with_peer( + my_policy: EncryptionPolicy, peer_policy: EncryptionPolicy | None +) -> PeerStore: + store = PeerStore(email="alice@example.com", encryption_policy=my_policy) + store.add_peer(Peer(email="bob@example.com", info=_peer_info(peer_policy))) + return store + + +def _keyed_pair( + alice_policy: EncryptionPolicy, bob_policy: EncryptionPolicy +) -> tuple[PeerStore, PeerStore]: + """Two stores that know each other's policy and hold each other's bundle.""" + alice = PeerStore(email="alice@example.com", encryption_policy=alice_policy) + bob = PeerStore(email="bob@example.com", encryption_policy=bob_policy) + alice.generate_keys() + bob.generate_keys() + alice.add_peer(Peer(email="bob@example.com", info=_peer_info(bob_policy))) + bob.add_peer(Peer(email="alice@example.com", info=_peer_info(alice_policy))) + alice.set_peer_bundle("bob@example.com", bob.get_public_bundle()) + bob.set_peer_bundle("alice@example.com", alice.get_public_bundle()) + return alice, bob + + +def _raw_inbox_message(do_manager, sender_email: str) -> bytes: + connection = do_manager.datasite_owner_syncer.connection_router + raw, _ = ( + connection.connection_for_receive_message().owner_download_next_raw_proposed_message_from_inbox( + sender_email + ) + ) + return raw + + +def _accepted_contents(do_manager) -> list[str]: + return [ + str(e.content) + for msg in do_manager._get_all_accepted_events_do() + for e in msg.events + ] + + +def _ds_cached_contents(ds_manager) -> list[str]: + cache = ds_manager.datasite_watcher_syncer.datasite_watcher_cache + return [str(e.content) for e in cache.get_cached_events()] + + +def _pair_with_stale_sender_view(): + """A required DO and an optional DS whose cached DO policy is outdated.""" + ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_encryption_policy=OPTIONAL, + do_encryption_policy=REQUIRED, + ) + grant_job_inbox_access(do_manager, ds_manager.email) + ds_manager.load_peers() + do_manager.load_peers() + stale = ds_manager.peer_manager.get_cached_peer(do_manager.email) + stale.info = _peer_info(OPTIONAL) + return ds_manager, do_manager + + +def _required_ds_pair(): + ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_encryption_policy=REQUIRED, + do_encryption_policy=OPTIONAL, + ) + grant_job_inbox_access(do_manager, ds_manager.email) + do_manager.load_peers() + return ds_manager, do_manager + + +def _owner_answers_in_plaintext(ds_manager, do_manager, content, monkeypatch): + """The owner handles a proposal and answers in plaintext, as a broken client would.""" + file_path = path_for_job(do_manager.email, ds_manager.email, "my.job") + ds_manager._send_file_change(file_path, content) + with monkeypatch.context() as m: + m.setattr(PeerStore, "encrypt_if_needed", lambda self, email, data: data) + do_manager.datasite_owner_syncer.sync([ds_manager.email]) + + +# ========================================================================= +# Policy model and advertisement +# ========================================================================= + + +def test_policy_defaults_to_optional(): + assert PeerStore(email="alice@example.com").encryption_policy == OPTIONAL + + +def test_policy_cannot_change_after_login(): + store = PeerStore(email="alice@example.com", encryption_policy=OPTIONAL) + with pytest.raises(ValidationError, match="frozen"): + store.encryption_policy = REQUIRED + + +@pytest.mark.parametrize("value, expected", [(True, REQUIRED), (False, OPTIONAL)]) +def test_encryption_setting_maps_to_policy(value, expected): + assert EncryptionPolicy.from_bool(value) == expected + + +def test_no_encryption_setting_maps_to_no_policy(): + assert EncryptionPolicy.from_bool(None) is None + + +@pytest.mark.parametrize("value", [OPTIONAL, "optional", 1]) +def test_encryption_setting_must_be_bool_or_none(value): + with pytest.raises(TypeError, match="True, False or None"): + EncryptionPolicy.from_bool(value) + + +def test_own_version_advertises_policy(): + ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_encryption_policy=OPTIONAL, + do_encryption_policy=REQUIRED, + ) + assert do_manager.peer_manager.get_own_peer_info().encryption_policy == REQUIRED + assert ds_manager.peer_manager.get_own_peer_info().encryption_policy == OPTIONAL + advertised = ds_manager.peer_manager.get_cached_peer(do_manager.email) + assert advertised.encryption_policy == REQUIRED + + +def test_missing_version_file_created_with_own_policy(): + a_conn, b_conn = mock_drive_service.pair_with_mock_service( + "a@example.com", "b@example.com" + ) + a_conn.share_peer_info_file_with_peer("b@example.com", _peer_info(REQUIRED)) + assert b_conn.read_peer_info_file("a@example.com").encryption_policy == REQUIRED + + +def test_malformed_peer_bundle_raises_on_peer_load(): + ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + add_peers=False, + do_encryption_policy=REQUIRED, + ) + ds_manager.add_peer(do_manager.email, sync=False) + do_manager.load_peers() + do_manager.approve_peer_request(ds_manager.email) + do_manager.peer_manager.connection_router.write_encryption_bundle( + ds_manager.email, "not json" + ) + with pytest.raises(ValueError): + ds_manager.load_peers() + + +# ========================================================================= +# Per-peer resolution +# ========================================================================= + + +@pytest.mark.parametrize( + "mine, theirs, encrypted", + [ + (OPTIONAL, OPTIONAL, False), + (OPTIONAL, REQUIRED, True), + (REQUIRED, OPTIONAL, True), + (REQUIRED, REQUIRED, True), + ], +) +def test_peer_encryption_resolved_from_both_policies(mine, theirs, encrypted): + store = _store_with_peer(mine, theirs) + assert store.pair_encrypts(store.get_cached_peer("bob@example.com")) is encrypted + + +def test_resolution_follows_changed_peer_policy(): + """An enclave that reboots with another setting is picked up on the next load.""" + store = _store_with_peer(OPTIONAL, OPTIONAL) + assert not store.pair_encrypts(store.get_cached_peer("bob@example.com")) + + store.get_cached_peer("bob@example.com").info = _peer_info(REQUIRED) + assert store.pair_encrypts(store.get_cached_peer("bob@example.com")) + + store.get_cached_peer("bob@example.com").info = _peer_info(OPTIONAL) + assert not store.pair_encrypts(store.get_cached_peer("bob@example.com")) + + +def test_sender_refuses_peer_with_unknown_policy(): + store = PeerStore(email="alice@example.com", encryption_policy=OPTIONAL) + store.add_peer(Peer(email="bob@example.com")) + with pytest.raises(SyftPeerNotReadyError, match="not known"): + store.encrypt_if_needed("bob@example.com", b"data") + + +def test_required_sender_encrypts_without_peer_policy(): + """A client that requires encryption encrypts whatever the peer advertises.""" + store = PeerStore(email="alice@example.com", encryption_policy=REQUIRED) + store.generate_keys() + store.add_peer(Peer(email="bob@example.com")) + with pytest.raises(SyftPeerNotReadyError, match="public encryption bundle"): + store.encrypt_if_needed("bob@example.com", b"data") + + +def test_unread_peer_info_policy_not_known(): + store = _store_with_peer(OPTIONAL, None) + with pytest.raises(SyftPeerNotReadyError, match="not known"): + store.encrypt_if_needed("bob@example.com", b"data") + + +def test_encrypting_pair_without_peer_bundle_not_ready(): + store = _store_with_peer(REQUIRED, OPTIONAL) + store.generate_keys() + with pytest.raises(SyftPeerNotReadyError): + store.encrypt_if_needed("bob@example.com", b"data") + + +# ========================================================================= +# Keys and bundles +# ========================================================================= + + +@pytest.mark.parametrize( + "requested, expected", [(None, OPTIONAL), (REQUIRED, REQUIRED)] +) +def test_first_client_writes_policy(tmp_path, requested, expected): + path = tmp_path / "syft_settings.json" + assert resolve_encryption_policy(path, requested) == expected + assert resolve_encryption_policy(path, None) == expected + + +def test_client_with_other_policy_than_disk_raises(tmp_path): + path = tmp_path / "syft_settings.json" + resolve_encryption_policy(path, OPTIONAL) + with pytest.raises(EncryptionPolicyMismatchError, match="'optional'"): + resolve_encryption_policy(path, REQUIRED) + + +def test_client_without_policy_takes_policy_of_syftbox(tmp_path): + """syft-bg starts after a notebook that required encryption.""" + + def start(policy): + config = PeerManagerConfig( + syftbox_folder=tmp_path, email="alice@example.com", encryption_policy=policy + ) + return PeerManager.from_config(config).peer_store.encryption_policy + + assert start(REQUIRED) == REQUIRED + assert start(None) == REQUIRED + + +def test_sync_raises_when_policy_on_disk_changed(): + """Another process deleted the SyftBox and started with another policy.""" + _, do_manager = SyftboxManager.pair_with_mock_drive_service_connection() + settings = datasite_settings_path(do_manager.syftbox_folder, do_manager.email) + settings.unlink() + resolve_encryption_policy(settings, REQUIRED) + with pytest.raises(EncryptionPolicyMismatchError): + do_manager.sync() + + +def test_policy_check_writes_missing_settings_back(): + """After delete_syftbox, the running client keeps its policy for the SyftBox.""" + _, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + do_encryption_policy=REQUIRED + ) + settings = datasite_settings_path(do_manager.syftbox_folder, do_manager.email) + settings.unlink() + do_manager.peer_manager.check_encryption_policy() + assert resolve_encryption_policy(settings, None) == REQUIRED + + +@pytest.mark.parametrize("policy", [OPTIONAL, REQUIRED]) +def test_every_policy_creates_key_file(tmp_path, policy): + keys_path = tmp_path / "crypto_keys.json" + store = PeerStore.create( + email="alice@example.com", encryption_policy=policy, keys_path=keys_path + ) + reloaded = PeerStore.create(email="alice@example.com", keys_path=keys_path) + assert reloaded.get_public_bundle() == store.get_public_bundle() + + +def test_plaintext_pair_publishes_bundles_at_peering(): + ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection() + ds_router = ds_manager.peer_manager.connection_router + do_router = do_manager.peer_manager.connection_router + assert ds_router.read_peer_encryption_bundle(do_manager.email) is not None + assert do_router.read_peer_encryption_bundle(ds_manager.email) is not None + + +def _publish_bundle_of_other_keys(do_manager, ds_manager) -> None: + """As a new runtime without the key file: Drive holds the bundle of older keys.""" + earlier = PeerStore(email=do_manager.email) + earlier.generate_keys() + do_manager.peer_manager.connection_router.write_encryption_bundle( + ds_manager.email, + json.dumps({"public_encryption_bundle": earlier.get_public_bundle()}), + ) + do_manager.peer_manager._first_load_done = False # as a new process + + +def test_first_peer_load_refuses_bundle_of_other_keys(): + ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + do_encryption_policy=REQUIRED, + ) + _publish_bundle_of_other_keys(do_manager, ds_manager) + with pytest.raises(ValueError, match="already has a key"): + do_manager.load_peers() + + +def test_bundle_of_other_keys_ignored_when_pair_does_not_encrypt(): + """No peer uses the bundle, so new keys on a new runtime are fine.""" + ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection() + _publish_bundle_of_other_keys(do_manager, ds_manager) + do_manager.load_peers() + + +# ========================================================================= +# Receive path +# ========================================================================= + + +@pytest.mark.parametrize( + "alice_policy, bob_policy", + [(REQUIRED, OPTIONAL), (OPTIONAL, REQUIRED)], + ids=["optional receiver", "required receiver"], +) +def test_receiver_rejects_plaintext_when_pair_encrypts(alice_policy, bob_policy): + """A pair that encrypts never sends plaintext, so plaintext is a downgrade.""" + _, bob = _keyed_pair(alice_policy, bob_policy) + with pytest.raises(ValueError, match="not encrypted"): + bob.decrypt_and_verify_if_needed("alice@example.com", b"plain") + + +def test_message_from_peer_with_unknown_policy_refused(): + _, bob = _keyed_pair(OPTIONAL, OPTIONAL) + bob.get_cached_peer("alice@example.com").info = None + with pytest.raises(SyftPeerNotReadyError, match="not known"): + bob.decrypt_and_verify_if_needed("alice@example.com", b"plain") + + +def test_receiver_accepts_plaintext_in_plaintext_pair(): + _, bob = _keyed_pair(OPTIONAL, OPTIONAL) + assert bob.decrypt_and_verify_if_needed("alice@example.com", b"plain") == b"plain" + + +def test_receiver_rejects_envelope_when_pair_does_not_encrypt(): + alice, bob = _keyed_pair(REQUIRED, OPTIONAL) + envelope = alice.encrypt("bob@example.com", b"secret") + bob.get_cached_peer("alice@example.com").info = _peer_info(OPTIONAL) + with pytest.raises(MessageFormatError, match="is encrypted"): + bob.decrypt_and_verify_if_needed("alice@example.com", envelope) + + +def test_envelope_without_peer_bundle_is_not_ready(): + alice, bob = _keyed_pair(REQUIRED, OPTIONAL) + envelope = alice.encrypt("bob@example.com", b"secret") + bob.get_cached_peer("alice@example.com").public_encryption_bundle = None + with pytest.raises(SyftPeerNotReadyError): + bob.decrypt_and_verify_if_needed("alice@example.com", envelope) + + +# ========================================================================= +# End to end over the mock drive +# ========================================================================= + + +@pytest.mark.parametrize( + "ds_policy, do_policy", + [(OPTIONAL, REQUIRED), (REQUIRED, OPTIONAL)], +) +def test_negotiated_encryption_flow(ds_policy, do_policy): + """The optional side complies with no manual configuration.""" + ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_encryption_policy=ds_policy, + do_encryption_policy=do_policy, + ) + grant_job_inbox_access(do_manager, ds_manager.email) + # One load_peers round each lets both sides see the policy and read bundles. + ds_manager.load_peers() + do_manager.load_peers() + + file_path = path_for_job(do_manager.email, ds_manager.email, "my.job") + ds_manager._send_file_change(file_path, "negotiated content") + assert PeerStore.is_envelope(_raw_inbox_message(do_manager, ds_manager.email)) + + do_manager.sync() + assert any("negotiated content" in c for c in _accepted_contents(do_manager)) + + ds_manager.sync() + assert len(_ds_cached_contents(ds_manager)) > 0 + + +def test_owner_send_to_non_peer_raises(): + _, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + do_encryption_policy=REQUIRED, + ) + syncer = do_manager.datasite_owner_syncer + message = FileChangeEventsMessage(events=[]) + syncer.outbox_queue.put(("former@example.com", message)) + with pytest.raises(SyftPeerNotReadyError, match="not known"): + syncer.process_syftbox_events_queue() + + +def test_owner_leaves_envelope_from_peer_without_bundle_in_inbox(): + """The sync goes on; the message stays in the inbox until the bundle is read.""" + ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_encryption_policy=REQUIRED, + do_encryption_policy=OPTIONAL, + ) + grant_job_inbox_access(do_manager, ds_manager.email) + do_manager.load_peers() + file_path = path_for_job(do_manager.email, ds_manager.email, "my.job") + ds_manager._send_file_change(file_path, "bundle later") + + peer = do_manager.peer_manager.peer_store.get_cached_peer(ds_manager.email) + peer.public_encryption_bundle = None + with pytest.warns(UserWarning, match="stay in the inbox"): + do_manager.datasite_owner_syncer.sync([ds_manager.email]) + assert not any("bundle later" in c for c in _accepted_contents(do_manager)) + + do_manager.sync() + assert any("bundle later" in c for c in _accepted_contents(do_manager)) + + +# ========================================================================= +# Sends that cannot go +# ========================================================================= + + +def _queued_changes(manager) -> int: + return manager.datasite_watcher_syncer.queue.qsize() + + +@pytest.mark.parametrize("do_policy", [OPTIONAL, REQUIRED]) +def test_send_before_owner_accepts_raises(do_policy): + """Before the owner accepts, its policy is not known, so nothing is sent.""" + ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + add_peers=False, + do_encryption_policy=do_policy, + ) + ds_manager.add_peer(do_manager.email, sync=False) + + file_path = path_for_job(do_manager.email, ds_manager.email, "my.job") + with pytest.raises(SyftPeerNotReadyError, match="not known"): + ds_manager._send_file_change(file_path, "early content") + assert _queued_changes(ds_manager) == 0 + + do_manager.load_peers() + do_manager.approve_peer_request(ds_manager.email) + grant_job_inbox_access(do_manager, ds_manager.email) + do_manager.sync() + assert not any("early content" in c for c in _accepted_contents(do_manager)) + + +def test_send_to_non_peer_raises(): + ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + add_peers=False, + ) + file_path = path_for_job(do_manager.email, ds_manager.email, "my.job") + with pytest.raises(SyftPeerNotReadyError, match="not known"): + ds_manager._send_file_change(file_path, "content") + + +def test_changes_for_two_datasites_go_to_each(monkeypatch): + ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection() + sent = [] + monkeypatch.setattr( + ConnectionRouter, + "watcher_send_proposed_file_changes_message", + lambda self, recipient, message: ( + sent.append( + (recipient, {fc.datasite_email for fc in message.proposed_file_changes}) + ) + ), + ) + watcher = ds_manager.datasite_watcher_syncer + watcher.on_file_change(f"{do_manager.email}/a.txt", "a", process_now=False) + watcher.on_file_change("other@example.com/b.txt", "b", process_now=False) + watcher.process_file_changes_queue() + assert sorted(sent) == sorted( + [ + (do_manager.email, {do_manager.email}), + ("other@example.com", {"other@example.com"}), + ] + ) + + +def _not_ready(recipient: str) -> SyftPeerNotReadyError: + return SyftPeerNotReadyError(recipient, cause="No bundle.", remedy="Sync.") + + +def test_failure_for_one_recipient_still_sends_to_other(monkeypatch): + ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection() + sent = [] + + def send_message(self, recipient, message): + if recipient == "other@example.com": + raise _not_ready(recipient) + sent.append(recipient) + + monkeypatch.setattr( + ConnectionRouter, "watcher_send_proposed_file_changes_message", send_message + ) + watcher = ds_manager.datasite_watcher_syncer + watcher.on_file_change("other@example.com/a.txt", "a", process_now=False) + watcher.on_file_change(f"{do_manager.email}/b.txt", "b", process_now=False) + with pytest.raises(SyftPeerNotReadyError): + watcher.process_file_changes_queue() + assert sent == [do_manager.email] + assert _queued_changes(ds_manager) == 0 + + +def test_unsent_owner_change_stays_pending(monkeypatch): + """The send raises, and the next sync sends the change.""" + ds_manager, do_manager = _pair_with_unsent_do_file() + + def fail(self, recipient, message): + raise _not_ready(recipient) + + with monkeypatch.context() as m: + m.setattr(ConnectionRouter, "owner_write_event_messages_to_outbox", fail) + with pytest.raises(SyftPeerNotReadyError): + do_manager.sync(auto_checkpoint=False) + + _assert_ds_receives_after_notebook_sync(ds_manager, do_manager) + + +# ========================================================================= +# Plaintext refused +# ========================================================================= + + +def test_plaintext_inbox_message_dropped_and_later_ones_read(): + ds_manager, do_manager = _pair_with_stale_sender_view() + file_path = path_for_job(do_manager.email, ds_manager.email, "my.job") + ds_manager._send_file_change(file_path, "plain content") + + ds_manager.load_peers() + # The owner never applies the dropped change, so this one is to another file. + other_path = path_for_job(do_manager.email, ds_manager.email, "other.job") + ds_manager._send_file_change(other_path, "encrypted content") + with pytest.warns(UserWarning, match="not encrypted"): + do_manager.sync() + contents = _accepted_contents(do_manager) + assert not any("plain content" in c for c in contents) + assert any("encrypted content" in c for c in contents) + + +def test_plaintext_outbox_message_skipped_and_later_ones_read(monkeypatch): + ds_manager, do_manager = _required_ds_pair() + _owner_answers_in_plaintext(ds_manager, do_manager, "plain content", monkeypatch) + + file_path = path_for_job(do_manager.email, ds_manager.email, "other.job") + ds_manager._send_file_change(file_path, "encrypted content") + do_manager.datasite_owner_syncer.sync([ds_manager.email]) + with pytest.warns(UserWarning, match="not encrypted"): + ds_manager.sync() + contents = _ds_cached_contents(ds_manager) + assert not any("plain content" in c for c in contents) + assert any("encrypted content" in c for c in contents) diff --git a/tests/unit/test_login_key_file.py b/tests/unit/test_login_key_file.py new file mode 100644 index 00000000000..c05272a469f --- /dev/null +++ b/tests/unit/test_login_key_file.py @@ -0,0 +1,38 @@ +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +import syft.sync.login as syft_login +from syft.sync.environments.environment import Environment +from syft.sync.peers.peer_store import PeerStore +from syft.sync.syftbox_manager import SyftboxManager + +EMAIL = "do@openmined.org" + + +@pytest.mark.parametrize("login", [syft_login.login_do, syft_login.login_ds]) +def test_login_restores_uploaded_key_file(tmp_path, monkeypatch, login): + uploaded = tmp_path / "upload" / "crypto_keys.json" + earlier = PeerStore.create(email=EMAIL, keys_path=uploaded) + + monkeypatch.setattr(Path, "home", lambda: tmp_path / "home") + monkeypatch.setattr(syft_login, "check_env", lambda: Environment.JUPYTER) + monkeypatch.setattr( + syft_login, + "handle_potential_version_mismatches_on_login", + lambda email, token_path: False, + ) + monkeypatch.setattr(syft_login, "_init_client_login", lambda *a, **kw: None) + configs = [] + monkeypatch.setattr( + SyftboxManager, + "from_config", + classmethod(lambda cls, config: configs.append(config) or MagicMock()), + ) + + login(EMAIL, token_path=tmp_path / "token.json", key_file=uploaded) + + keys_path = configs[0].peer_manager_config.crypto_keys_path + restored = PeerStore.create(email=EMAIL, keys_path=keys_path) + assert restored.get_public_bundle() == earlier.get_public_bundle() diff --git a/tests/unit/test_peer_requests.py b/tests/unit/test_peer_requests.py index 3d74b428381..3ce239fc408 100644 --- a/tests/unit/test_peer_requests.py +++ b/tests/unit/test_peer_requests.py @@ -1,5 +1,8 @@ from pathlib import Path +import pytest +from syft.sync.peers.exceptions import SyftPeerNotReadyError + from syft.sync.syftbox_manager import SyftboxManager from tests.unit.test_sync_manager import path_for_job from tests.unit.utils import grant_job_inbox_access @@ -11,10 +14,10 @@ def test_peer_request_blocks_sync_until_approved(): Workflow: 1. DS adds DO as peer (creates peer request) - 2. DS submits a job + 2. DS submits a job - the send raises (DO policy not known) 3. DO syncs - nothing should sync (peer not approved) 4. DO approves peer request - 5. DO syncs - job should now sync + 5. DS syncs and submits again, DO syncs - job should now sync """ # Create managers with mock drive service connection, no auto-add peers # Note: email1 becomes DO, email2 becomes DS @@ -36,7 +39,8 @@ def test_peer_request_blocks_sync_until_approved(): # Step 2: DS submits a file to their job folder job_file_path = path_for_job(do_manager.email, ds_manager.email) job_content = "print('Hello from DS')" - ds_manager._send_file_change(job_file_path, job_content) + with pytest.raises(SyftPeerNotReadyError, match="not known"): + ds_manager._send_file_change(job_file_path, job_content) # Step 3: DO syncs WITHOUT accepting - nothing should sync do_manager.sync() @@ -54,7 +58,10 @@ def test_peer_request_blocks_sync_until_approved(): assert len(do_manager.peer_manager.approved_peers) == 1 assert do_manager.peer_manager.approved_peers[0].email == ds_manager.email - # Step 5: DO syncs again - now it should work + # Step 5: DS syncs, which reads the DO policy, and submits again. DO syncs + # again - now it should work + ds_manager.sync() + ds_manager._send_file_change(job_file_path, job_content) do_manager.sync() # Verify: File synced and in cache @@ -85,9 +92,10 @@ def test_peer_request_rejection(): ds_manager.add_peer(do_manager.email) do_manager.load_peers() - # DS sends a job + # DS sends a job; the DO policy is not known before it accepts job_file_path = path_for_job(do_manager.email, ds_manager.email) - ds_manager._send_file_change(job_file_path, "print('test')") + with pytest.raises(SyftPeerNotReadyError): + ds_manager._send_file_change(job_file_path, "print('test')") # DO rejects the peer request do_manager.reject_peer_request(ds_manager.email) diff --git a/tests/unit/test_persisted_cache.py b/tests/unit/test_persisted_cache.py index ab1b31658b6..7d9c0f49246 100644 --- a/tests/unit/test_persisted_cache.py +++ b/tests/unit/test_persisted_cache.py @@ -151,7 +151,7 @@ def test_peer_gets_file_when_other_process_cannot_read_peer_version(monkeypatch) do_manager_b = do_manager._copy() monkeypatch.setattr( do_manager_b.peer_manager, - "_load_single_peer_version", + "_load_single_peer_info", lambda email: (email, None), ) do_manager_b.sync(auto_checkpoint=False) @@ -189,7 +189,7 @@ def test_peer_loses_file_deleted_while_other_process_skipped_it(monkeypatch): do_manager_b = do_manager._copy() monkeypatch.setattr( do_manager_b.peer_manager, - "_load_single_peer_version", + "_load_single_peer_info", lambda email: (email, None), ) do_manager_b.sync(auto_checkpoint=False) @@ -212,7 +212,7 @@ def test_read_error_on_pending_file_raises_and_keeps_peer_copy(monkeypatch): do_manager_b = do_manager._copy() monkeypatch.setattr( do_manager_b.peer_manager, - "_load_single_peer_version", + "_load_single_peer_info", lambda email: (email, None), ) do_manager_b.sync(auto_checkpoint=False) diff --git a/tests/unit/test_sync_manager.py b/tests/unit/test_sync_manager.py index 1227b26fb8f..cfcd0036905 100644 --- a/tests/unit/test_sync_manager.py +++ b/tests/unit/test_sync_manager.py @@ -5,10 +5,14 @@ from syft.sync.connections.connection_router import ConnectionRouter from syft.sync.connections.drive.gdrive_transport import GDriveConnection +from syft.sync.encryption_policy import EncryptionPolicy +from syft.sync.peers.peer import Peer from syft.sync.peers.peer_store import PeerStore from syft.sync.messages.proposed_filechange import ProposedFileChangesMessage from syft.sync.messages.proposed_filechange import ProposedFileChange from syft.sync.syftbox_manager import SyftboxManager +from syft.sync.version.peer_info import PeerInfo +from syft.sync.version.version_info import VersionInfo from syft.sync.sync.caches.datasite_owner_cache import ( ProposedEventFileOutdatedException, ) @@ -164,7 +168,8 @@ def test_sync_existing_datasite_state_do(): n_outbox = connection_ds.watcher_get_outbox_file_metadatas(do_manager.email, None) assert n_messages_in_cache >= 1 # At least 1 message with the 2 file changes - assert n_files_in_cache == 2 # 2 data files + # 2 data files + private/crypto_keys.json + private/syft_settings.json + assert n_files_in_cache == 4 assert hashes_in_cache == 2 # 2 data files assert len(n_outbox) >= 1 @@ -200,7 +205,9 @@ def test_sync_existing_inbox_state_do(): n_files_in_cache = len(do_manager.datasite_owner_syncer.event_cache.file_connection) hashes_in_cache = len(do_manager.datasite_owner_syncer.event_cache.file_hashes) assert n_events_message_in_cache >= 1 # At least 1 message with 2 file changes - assert n_files_in_cache == 3 # 2 data files + root syft.pub.yaml + # 2 data files + root syft.pub.yaml + private/crypto_keys.json + # + private/syft_settings.json + assert n_files_in_cache == 5 assert hashes_in_cache == 3 # 2 data files + root syft.pub.yaml @@ -434,6 +441,26 @@ def test_collection_files_excluded_from_outbox_sync(): ) +def _store_knowing(email: str, peer_email: str) -> PeerStore: + """A store for ``email`` that knows ``peer_email`` as an optional peer.""" + info = PeerInfo( + version_info=VersionInfo.current(), + encryption_policy=EncryptionPolicy.OPTIONAL, + ) + store = PeerStore(email=email) + store.add_peer(Peer(email=peer_email, info=info)) + return store + + +def _add_known_peer(do_manager, email: str) -> None: + """Make ``email`` a peer of the DO with a known version, as peering does.""" + info = PeerInfo( + version_info=VersionInfo.current(), + encryption_policy=EncryptionPolicy.OPTIONAL, + ) + do_manager.peer_manager.peer_store.add_peer(Peer(email=email, info=info)) + + def test_job_files_sync_to_submitter_only(): """Test that job files only sync to the peer who submitted the job. @@ -487,9 +514,10 @@ def test_job_files_sync_to_submitter_only(): submitter_email, ds_manager._connection_router.connections[0].drive_service ) submitter_conn.add_peer(do_manager.email) + _add_known_peer(do_manager, submitter_email) submitter_router = ConnectionRouter( connections=[submitter_conn], - peer_store=PeerStore(email=submitter_email), + peer_store=_store_knowing(submitter_email, do_manager.email), ) do_manager.datasite_owner_syncer.process_local_changes(recipients) @@ -689,9 +717,10 @@ def test_permission_change_triggers_resend(): peer_b_email, ds_manager._connection_router.connections[0].drive_service ) peer_b_conn.add_peer(do_manager.email) + _add_known_peer(do_manager, peer_b_email) peer_b_router = ConnectionRouter( connections=[peer_b_conn], - peer_store=PeerStore(email=peer_b_email), + peer_store=_store_knowing(peer_b_email, do_manager.email), ) # Grant only peer A read access to project/ diff --git a/tests/unit/test_validate_peer.py b/tests/unit/test_validate_peer.py index 1610cdababc..a22debf6295 100644 --- a/tests/unit/test_validate_peer.py +++ b/tests/unit/test_validate_peer.py @@ -72,7 +72,7 @@ def test_reset_then_re_peer_in_the_same_session_is_valid(recwarn): ds_manager, do_manager = _pair(add_peers=True) # connected at start for manager in (ds_manager, do_manager): manager.delete_syftbox(verbose=False, broadcast_delete_events=False) - manager.peer_manager.write_own_version() + manager.peer_manager.write_own_peer_info() ds_manager.add_peer(do_manager.email) do_manager.approve_peer_request(ds_manager.email, peer_must_exist=False) diff --git a/tests/unit/test_version_negotiation.py b/tests/unit/test_version_negotiation.py index 32336768b09..a46f144c99e 100644 --- a/tests/unit/test_version_negotiation.py +++ b/tests/unit/test_version_negotiation.py @@ -7,6 +7,8 @@ from syft.sync.version.exceptions import ( VersionMismatchError, ) +from syft.sync.encryption_policy import EncryptionPolicy +from syft.sync.version.peer_info import PeerInfo from syft.sync.version.peer_manager import CompatAction from syft.sync.version.version_info import CompatibilityStatus, VersionInfo @@ -165,8 +167,8 @@ def test_peer_manager_writes_own_version(self): SyftboxManager.pair_with_mock_drive_service_connection() ) - ds_version = do_manager.peer_manager.load_peer_version(ds_manager.email) - do_version = ds_manager.peer_manager.load_peer_version(do_manager.email) + ds_version = do_manager.peer_manager.load_peer_info(ds_manager.email) + do_version = ds_manager.peer_manager.load_peer_info(do_manager.email) assert ds_version is not None assert do_version is not None @@ -176,12 +178,12 @@ def test_version_shared_on_add_peer(self): add_peers=False ) - ds_version_before = do_manager.peer_manager.load_peer_version(ds_manager.email) + ds_version_before = do_manager.peer_manager.load_peer_info(ds_manager.email) assert ds_version_before is None ds_manager.add_peer(do_manager.email) - ds_version_after = do_manager.peer_manager.load_peer_version(ds_manager.email) + ds_version_after = do_manager.peer_manager.load_peer_info(ds_manager.email) assert ds_version_after is not None def test_version_shared_on_approve_peer(self): @@ -192,12 +194,12 @@ def test_version_shared_on_approve_peer(self): ds_manager.add_peer(do_manager.email) do_manager.load_peers() - do_version_before = ds_manager.peer_manager.load_peer_version(do_manager.email) + do_version_before = ds_manager.peer_manager.load_peer_info(do_manager.email) assert do_version_before is None do_manager.approve_peer_request(ds_manager.email) - do_version_after = ds_manager.peer_manager.load_peer_version(do_manager.email) + do_version_after = ds_manager.peer_manager.load_peer_info(do_manager.email) assert do_version_after is not None def test_load_peer_version(self): @@ -205,16 +207,19 @@ def test_load_peer_version(self): SyftboxManager.pair_with_mock_drive_service_connection() ) - version = ds_manager.peer_manager.load_peer_version(do_manager.email) - assert version is not None - assert version.syft_client_version == VersionInfo.current().syft_client_version + info = ds_manager.peer_manager.load_peer_info(do_manager.email) + assert info is not None + assert ( + info.version_info.syft_client_version + == VersionInfo.current().syft_client_version + ) def test_load_peer_version_without_permission(self): ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( add_peers=False ) - version = ds_manager.peer_manager.load_peer_version(do_manager.email) + version = ds_manager.peer_manager.load_peer_info(do_manager.email) assert version is None def test_load_peer_versions_parallel(self): @@ -235,13 +240,16 @@ def test_load_peer_versions_parallel(self): ].drive_service._backing_store third_peer_email = "third_peer@test.com" - third_peer_version = VersionInfo.current() + third_peer_info = PeerInfo( + version_info=VersionInfo.current(), + encryption_policy=EncryptionPolicy.OPTIONAL, + ) version_file = MockDriveFile( name=SYFT_VERSION_FILE, mimeType="application/json", parents=[], owners=[{"emailAddress": third_peer_email}], - content=third_peer_version.to_json(), + content=third_peer_info.to_json(), ) backing_store.add_file(version_file) @@ -255,7 +263,7 @@ def test_load_peer_versions_parallel(self): ) peer_emails = [do_manager.email, third_peer_email] - versions = ds_manager.peer_manager.load_peer_versions_parallel( + versions = ds_manager.peer_manager.load_peer_infos_parallel( peer_emails, force=True ) @@ -263,8 +271,8 @@ def test_load_peer_versions_parallel(self): assert versions[do_manager.email] is not None assert versions[third_peer_email] is not None assert ( - versions[third_peer_email].syft_client_version - == third_peer_version.syft_client_version + versions[third_peer_email].version_info.syft_client_version + == third_peer_info.version_info.syft_client_version ) @@ -272,7 +280,8 @@ def _set_peer_version(manager: SyftboxManager, peer_email: str, version: Version """Override the cached version for a peer without a Drive round-trip.""" peer = manager.peer_manager.get_cached_peer(peer_email) assert peer is not None, f"peer {peer_email} not in store" - peer.version = version + policy = peer.encryption_policy or EncryptionPolicy.OPTIONAL + peer.info = PeerInfo(version_info=version, encryption_policy=policy) def _patch_plus_one_version() -> VersionInfo: @@ -502,8 +511,7 @@ def test_sync_skips_a_peer_of_unknown_version(self): ) peer = do_manager.peer_manager.get_cached_peer(ds_manager.email) assert peer is not None - peer.version = None - do_manager.peer_manager._loaded_peer_versions[ds_manager.email] = None + peer.info = None do_manager.peer_manager.suppress_version_warnings = True compatible_peers = ( @@ -519,8 +527,8 @@ def test_version_upgrade_breaks_communication(self): check_versions=True, ) - ds_manager.peer_manager.load_peer_version(do_manager.email) - do_manager.peer_manager.load_peer_version(ds_manager.email) + ds_manager.peer_manager.load_peer_info(do_manager.email) + do_manager.peer_manager.load_peer_info(ds_manager.email) assert ds_manager.peer_manager.is_peer_version_compatible(do_manager.email) assert do_manager.peer_manager.is_peer_version_compatible(ds_manager.email)