Merge branch 'master' into project-history

Author: ValentinBuira

.github/workflows/autotests.yml

@@ -1,7 +1,7 @@
 name: Auto Tests
 on: [push]
 env:
-  TEST_MERGIN_URL: https://test.dev.merginmaps.com/
+  TEST_MERGIN_URL: https://app.dev.merginmaps.com/
   TEST_API_USERNAME: test_plugin
   TEST_API_PASSWORD: ${{ secrets.MERGINTEST_API_PASSWORD }}
   TEST_API_USERNAME2: test_plugin2

README.md

@@ -71,6 +71,7 @@ Commands:
   download             Download last version of Mergin Maps project
   download-file        Download project file at specified version.
   list-projects        List projects on the server
+  list-files           List files in a project
   login                Login to the service and see how to set the token...
   pull                 Fetch changes from Mergin Maps repository
   push                 Upload local changes into Mergin Maps repository

mergin/cli.py

@@ -7,7 +7,7 @@
 pip puts these tools).
 """
 
-from datetime import datetime, timezone
+from datetime import datetime, timezone, date
 import click
 import json
 import os
@@ -143,7 +143,7 @@ def _print_unhandled_exception():
 
 
 @click.group(
-    epilog=f"Copyright (C) 2019-2021 Lutra Consulting\n\n(mergin-py-client v{__version__} / pygeodiff v{GeoDiff().version()})"
+    epilog=f"Copyright (C) 2019-{date.today().year} Lutra Consulting\n\n(mergin-py-client v{__version__} / pygeodiff v{GeoDiff().version()})"
 )
 @click.option(
     "--url",
@@ -481,9 +481,9 @@ def show_version(ctx, version):
         return
     directory = os.getcwd()
     mp = MerginProject(directory)
-    project_path = mp.project_full_name()
+    project_id = mp.project_id()
     # TODO: handle exception when version not found
-    version_info_dict = mc.project_version_info(project_path, version)[0]
+    version_info_dict = mc.project_version_info(project_id, version)
     click.secho("Project: " + version_info_dict["namespace"] + "/" + version_info_dict["project_name"])
     click.secho("Version: " + version_info_dict["name"] + " by " + version_info_dict["author"])
     click.secho("Time:    " + version_info_dict["created"])
@@ -661,6 +661,27 @@ def reset(ctx):
     except Exception as e:
         _print_unhandled_exception()
 
+@cli.command()
+@click.argument("project")
+@click.option("--json", is_flag=True, default=False, help="Output in JSON format")
+@click.pass_context
+def list_files(ctx, project, json):
+    """List files in a project."""
+
+    mc = ctx.obj["client"]
+    if mc is None:
+        return
+
+    project_info = mc.project_info(project)
+    project_files = project_info["files"]
+
+    if json:
+        print(project_files)
+    else:
+        click.echo("Fetched {} files .".format(len(project_files)))
+        for file in project_files:
+            click.echo("  {:40}\t{:6.1f} MB".format(file['path'], file["size"] / (1024 * 1024)))
+
 
 if __name__ == "__main__":
     cli()

mergin/client.py

@@ -17,7 +17,7 @@
 import typing
 import warnings
 
-from .common import ClientError, LoginError, InvalidProject
+from .common import ClientError, LoginError, InvalidProject, ErrorCode
 from .merginproject import MerginProject
 from .client_pull import (
     download_file_finalize,
@@ -205,19 +205,20 @@ def _do_request(self, request):
         try:
             return self.opener.open(request)
         except urllib.error.HTTPError as e:
-            if e.headers.get("Content-Type", "") == "application/problem+json":
-                info = json.load(e)
-                err_detail = info.get("detail")
-            else:
-                err_detail = e.read().decode("utf-8")
+            server_response = json.load(e)
 
-            error_msg = (
-                f"HTTP Error: {e.code} {e.reason}\n"
-                f"URL: {request.get_full_url()}\n"
-                f"Method: {request.get_method()}\n"
-                f"Detail: {err_detail}"
+            # We first to try to get the value from the response otherwise we set a default value
+            err_detail = server_response.get("detail", e.read().decode("utf-8"))
+            server_code = server_response.get("code", None)
+
+            raise ClientError(
+                detail=err_detail,
+                url=request.get_full_url(),
+                server_code=server_code,
+                server_response=server_response,
+                http_error=e.code,
+                http_method=request.get_method(),
             )
-            raise ClientError(error_msg)
         except urllib.error.URLError as e:
             # e.g. when DNS resolution fails (no internet connection?)
             raise ClientError("Error requesting " + request.full_url + ": " + str(e))
@@ -447,9 +448,9 @@ def create_workspace(self, workspace_name):
 
         try:
             self.post("/v1/workspace", params, {"Content-Type": "application/json"})
-        except Exception as e:
-            detail = f"Workspace name: {workspace_name}"
-            raise ClientError(str(e), detail)
+        except ClientError as e:
+            e.extra = f"Workspace name: {workspace_name}"
+            raise e
 
     def create_project(self, project_name, is_public=False, namespace=None):
         """
@@ -496,9 +497,9 @@ def create_project(self, project_name, is_public=False, namespace=None):
             namespace = self.username()
         try:
             self.post(f"/v1/project/{namespace}", params, {"Content-Type": "application/json"})
-        except Exception as e:
-            detail = f"Namespace: {namespace}, project name: {project_name}"
-            raise ClientError(str(e), detail)
+        except ClientError as e:
+            e.extra = f"Namespace: {namespace}, project name: {project_name}"
+            raise e
 
     def create_project_and_push(self, project_name, directory, is_public=False, namespace=None):
         """
@@ -833,9 +834,9 @@ def set_project_access(self, project_path, access):
         try:
             request = urllib.request.Request(url, data=json.dumps(params).encode(), headers=json_headers, method="PUT")
             self._do_request(request)
-        except Exception as e:
-            detail = f"Project path: {project_path}"
-            raise ClientError(str(e), detail)
+        except ClientError as e:
+            e.extra = f"Project path: {project_path}"
+            raise e
 
     def add_user_permissions_to_project(self, project_path, usernames, permission_level):
         """
@@ -1040,13 +1041,10 @@ def project_status(self, directory):
 
         return pull_changes, push_changes, push_changes_summary
 
-    def project_version_info(self, project_path, version):
+    def project_version_info(self, project_id, version):
         """Returns JSON with detailed information about a single project version"""
-        params = {"version_id": version}
-        params = {"page": version, "per_page": 1, "descending": False}
-        resp = self.get(f"/v1/project/versions/paginated/{project_path}", params)
-        j = json.load(resp)
-        return j["versions"]
+        resp = self.get(f"/v1/project/version/{project_id}/{version}")
+        return json.load(resp)
 
     def project_file_history_info(self, project_path, file_path):
         """Returns JSON with full history of a single file within a project"""

mergin/common.py

@@ -1,5 +1,5 @@
 import os
-
+from enum import Enum
 
 CHUNK_SIZE = 100 * 1024 * 1024
 
@@ -10,8 +10,35 @@
 this_dir = os.path.dirname(os.path.realpath(__file__))
 
 
+# Error code from the public API, add to the end of enum as we handle more eror
+class ErrorCode(Enum):
+    ProjectsLimitHit = "ProjectsLimitHit"
+    StorageLimitHit = "StorageLimitHit"
+
+
 class ClientError(Exception):
-    pass
+    def __init__(self, detail, url=None, server_code=None, server_response=None, http_error=None, http_method=None):
+        self.detail = detail
+        self.url = url
+        self.http_error = http_error
+        self.http_method = http_method
+
+        self.server_code = server_code
+        self.server_response = server_response
+
+        self.extra = None
+
+    def __str__(self):
+        string_res = f"Detail: {self.detail}\n"
+        if self.http_error:
+            string_res += f"HTTP Error: {self.http_error}\n"
+        if self.url:
+            string_res += f"URL: {self.url}\n"
+        if self.http_method:
+            string_res += f"Method: {self.http_method}\n"
+        if self.extra:
+            string_res += f"{self.extra}\n"
+        return string_res
 
 
 class LoginError(Exception):

mergin/test/test_client.py

@@ -19,6 +19,7 @@
     decode_token_data,
     TokenError,
     ServerType,
+    ErrorCode,
 )
 from ..client_push import push_project_async, push_project_cancel
 from ..client_pull import (
@@ -224,10 +225,9 @@ def test_create_remote_project_from_local(mc):
     assert project_info["namespace"] == API_USER
     assert project_info["id"] == source_mp.project_id()
 
-    versions = mc.project_versions(project)
-    assert len(versions) == 1
-    assert versions[0]["name"] == "v1"
-    assert any(f for f in versions[0]["changes"]["added"] if f["path"] == "test.qgs")
+    version = mc.project_version_info(project_info.get("id"), "v1")
+    assert version["name"] == "v1"
+    assert any(f for f in version["changes"]["added"] if f["path"] == "test.qgs")
 
     # check we can fully download remote project
     mc.download_project(project, download_dir)
@@ -305,13 +305,9 @@ def test_push_pull_changes(mc):
     assert generate_checksum(os.path.join(project_dir, f_updated)) == f_remote_checksum
     assert project_info["id"] == mp.project_id()
     assert len(project_info["files"]) == len(mp.inspect_files())
-    project_versions = mc.project_versions(project)
-    assert len(project_versions) == 2
-    f_change = next(
-        (f for f in project_versions[-1]["changes"]["updated"] if f["path"] == f_updated),
-        None,
-    )
-    assert "origin_checksum" not in f_change  # internal client info
+    project_version = mc.project_version_info(project_info["id"], "v2")
+    updated_file = [f for f in project_version["changes"]["updated"] if f["path"] == f_updated][0]
+    assert "origin_checksum" not in updated_file  # internal client info
 
     # test parallel changes
     with open(os.path.join(project_dir_2, f_updated), "w") as f:
@@ -796,7 +792,7 @@ def test_available_storage_validation(mcStorage):
     assert got_right_err
 
     # Expecting empty project
-    project_info = get_project_info(mcStorage, API_USER, test_project)
+    project_info = get_project_info(mcStorage, STORAGE_WORKSPACE, test_project)
     assert project_info["version"] == "v0"
     assert project_info["disk_usage"] == 0
 
@@ -2018,7 +2014,7 @@ def test_report(mc):
             ]
         )
         assert headers in content
-        assert "base.gpkg,simple,test_plugin" in content
+        assert f"base.gpkg,simple,{API_USER}" in content
         assert "v3,update,,,2" in content
         # files not edited are not in reports
         assert "inserted_1_A.gpkg" not in content
@@ -2163,7 +2159,7 @@ def test_version_info(mc):
     project = API_USER + "/" + test_project
     project_dir = os.path.join(TMP_DIR, test_project)  # primary project dir
     test_gpkg = "test.gpkg"
-    file_path = os.path.join(project_dir, "test.gpkg")
+    file_path = os.path.join(project_dir, test_gpkg)
 
     cleanup(mc, project, [project_dir])
 
@@ -2176,8 +2172,8 @@ def test_version_info(mc):
 
     shutil.copy(os.path.join(TEST_DATA_DIR, "inserted_1_A_mod.gpkg"), file_path)
     mc.push_project(project_dir)
-
-    info = mc.project_version_info(project, 2)[0]
+    project_info = mc.project_info(project)
+    info = mc.project_version_info(project_info.get("id"), "v2")
     assert info["namespace"] == API_USER
     assert info["project_name"] == test_project
     assert info["name"] == "v2"
@@ -2344,9 +2340,14 @@ def test_project_metadata(mc):
 
     # copy metadata in old format
     os.makedirs(os.path.join(project_dir, ".mergin"), exist_ok=True)
-    project_metadata = os.path.join(project_dir, ".mergin", "mergin.json")
     metadata_file = os.path.join(project_dir, "old_metadata.json")
-    shutil.copyfile(metadata_file, project_metadata)
+    # rewrite metadata nemespace to prevent failing tests with other user than test_plugin
+    with open(metadata_file, "r") as f:
+        metadata = json.load(f)
+    metadata["name"] = f"{API_USER}/{test_project}"
+    project_metadata_file = os.path.join(project_dir, ".mergin", "mergin.json")
+    with open(project_metadata_file, "w") as f:
+        json.dump(metadata, f, indent=2)
 
     # verify we have correct metadata
     mp = MerginProject(project_dir)
@@ -2357,7 +2358,12 @@ def test_project_metadata(mc):
 
     # copy metadata in new format
     metadata_file = os.path.join(project_dir, "new_metadata.json")
-    shutil.copyfile(metadata_file, project_metadata)
+    # rewrite metadata nemespace to prevent failing tests with other user than test_plugin
+    with open(metadata_file, "r") as f:
+        metadata = json.load(f)
+    metadata["namespace"] = API_USER
+    with open(project_metadata_file, "w") as f:
+        json.dump(metadata, f, indent=2)
 
     # verify we have correct metadata
     mp = MerginProject(project_dir)
@@ -2629,3 +2635,33 @@ def test_editor_push(mc: MerginClient, mc2: MerginClient):
             conflicted_file = project_file
     # There is no conflicted qgs file
     assert conflicted_file is None
+
+
+def test_error_push_already_named_project(mc: MerginClient):
+    test_project = "test_push_already_existing"
+    project_dir = os.path.join(TMP_DIR, test_project)
+
+    with pytest.raises(ClientError) as e:
+        mc.create_project_and_push(test_project, project_dir)
+    assert e.value.detail == "Project with the same name already exists"
+    assert e.value.http_error == 409
+    assert e.value.http_method == "POST"
+    assert e.value.url == f"{mc.url}v1/project/{API_USER}"
+
+
+def test_error_projects_limit_hit(mcStorage: MerginClient):
+    test_project = "test_another_project_above_projects_limit"
+    test_project_fullname = STORAGE_WORKSPACE + "/" + test_project
+
+    project_dir = os.path.join(TMP_DIR, test_project, API_USER)
+
+    with pytest.raises(ClientError) as e:
+        mcStorage.create_project_and_push(test_project_fullname, project_dir)
+    assert e.value.server_code == ErrorCode.ProjectsLimitHit.value
+    assert (
+        e.value.detail
+        == "Maximum number of projects is reached. Please upgrade your subscription to create new projects (ProjectsLimitHit)"
+    )
+    assert e.value.http_error == 422
+    assert e.value.http_method == "POST"
+    assert e.value.url == f"{mcStorage.url}v1/project/testpluginstorage"

mergin/utils.py

@@ -275,3 +275,22 @@ def is_mergin_config(path: str) -> bool:
     """Check if the given path is for file mergin-config.json"""
     filename = os.path.basename(path).lower()
     return filename == "mergin-config.json"
+
+
+def bytes_to_human_size(bytes: int):
+    """
+    Convert bytes to human readable size
+    example  :
+    bytes_to_human_size(5600000) -> "5.3 MB"
+    """
+    precision = 1
+    if bytes < 1e-5:
+        return "0.0 MB"
+    elif bytes < 1024.0 * 1024.0:
+        return f"{round( bytes / 1024.0, precision )} KB"
+    elif bytes < 1024.0 * 1024.0 * 1024.0:
+        return f"{round( bytes / 1024.0 / 1024.0, precision)} MB"
+    elif bytes < 1024.0 * 1024.0 * 1024.0 * 1024.0:
+        return f"{round( bytes / 1024.0 / 1024.0 / 1024.0, precision )} GB"
+    else:
+        return f"{round( bytes / 1024.0 / 1024.0 / 1024.0 / 1024.0, precision )} TB"