new CLI parameter "unordered"

Author: mtmail

README.md

@@ -131,6 +131,7 @@ options:
   --api-domain          API domain (default api.opencagedata.com)
   --optional-api-params
                         Extra parameters for each request (e.g. language=fr,no_dedupe=1)
+  --unordered           Allow the output lines to be in different order (can be faster)
   --limit               Stop after this number of lines in the input
   --dry-run             Read the input file but no geocoding
   --no-progress         Display no progress bar

opencage/batch.py

@@ -191,13 +191,14 @@ async def write_one_geocoding_result(self, output, row_id, address, geocoding_re
         # Enforce that row are written ordered. That means we might wait for other threads
         # to finish a task and make the overall process slower. Alternative would be to
         # use a second queue, or keep some results in memory.
-        while row_id > self.write_counter:
-            if self.options.verbose:
-                self.log(f"Want to write row {row_id}, but write_counter is at {self.write_counter}")
-            await asyncio.sleep(random.uniform(0.01, 0.1))
+        if not self.options.unordered:
+            while row_id > self.write_counter:
+                if self.options.verbose:
+                    self.log(f"Want to write row {row_id}, but write_counter is at {self.write_counter}")
+                await asyncio.sleep(random.uniform(0.01, 0.1))
 
-        if self.options.verbose:
-            self.log(f"Writing row {row_id}")
+            if self.options.verbose:
+                self.log(f"Writing row {row_id}")
         output.writerow(row)
         self.write_counter = self.write_counter + 1
 

opencage/command_line.py

@@ -68,6 +68,7 @@ def add_optional_arguments(parser):
     parser.add_argument("--api-domain", type=str, default="api.opencagedata.com", help="API domain (default api.opencagedata.com)", metavar='')
     parser.add_argument("--optional-api-params", type=comma_separated_dict_type, default="", help="Extra parameters for each request (e.g. language=fr,no_dedupe=1)", metavar='')
     parser.add_argument("--limit", type=int, default=0, help="Stop after this number of lines in the input", metavar='')
+    parser.add_argument("--unordered", action="store_true", help="Allow the output lines to be in different order (can be faster)")
     parser.add_argument("--dry-run", action="store_true", help="Read the input file but no geocoding")
     parser.add_argument("--no-progress", action="store_true", help="Display no progress bar")
     parser.add_argument("--quiet", action="store_true", help="No progress bar and no messages")

test/cli/test_cli_args.py

@@ -93,6 +93,7 @@ def test_full_argument_list():
         "--timeout", "2",
         "--retries", "1",
         "--dry-run",
+        "--unordered",
         "--api-domain", "bulk.opencagedata.com",
         "--optional-api-params", "extra=1",
         "--no-progress",
@@ -111,6 +112,7 @@ def test_full_argument_list():
     assert args.timeout == 2
     assert args.retries == 1
     assert args.dry_run is True
+    assert args.unordered is True
     assert args.api_domain == "bulk.opencagedata.com"
     assert args.optional_api_params == { "extra": "1" }
     assert args.no_progress is True
@@ -133,6 +135,7 @@ def test_defaults():
     assert args.timeout == 10
     assert args.retries == 10
     assert args.dry_run is False
+    assert args.unordered is False
     assert args.api_domain == "api.opencagedata.com"
     assert args.optional_api_params == {}
     assert args.no_progress is False