allow setting ssl context in case of SSL certificate errors (#52)

Author: mtmail

Changes.txt

@@ -1,6 +1,10 @@
-unreleased
+v2.3.1 Wed Nov 15 2023
+  New error 'SSLError' which is more explicit in case of SSL certificate chain issues
+  Allow setting a domain name (only used in test suite)
+  Allow setting sslcontext, for example to ignore SSL certificat errors (for debugging only)
   Batch example: Guess if input is coordinate pair, if so then do reverse geocoding
   Batch example: Give example of input file format
+  Batch example: Ship CA root certificates instead of relying on those of the operating system
 
 v2.3.0 Tue 04 Jul 2023
   Batch example: Raise exception when API key fails (quota, missing API key)

examples/batch.py

@@ -32,12 +32,21 @@
 import sys
 import csv
 import re
+import ssl
 import asyncio
 import traceback
+import aiohttp
 import backoff
+import certifi
 from tqdm import tqdm
-from opencage.geocoder import OpenCageGeocode, AioHttpError
+from opencage.geocoder import OpenCageGeocode, SSLError
 
+# Use certificates from the certifi package instead of those of the operating system
+# https://pypi.org/project/certifi/
+# https://docs.aiohttp.org/en/stable/client_advanced.html#ssl-control-for-tcp-sockets
+sslcontext = ssl.create_default_context(cafile=certifi.where())
+# Alternatively set sslcontext=False to ignore certificate validation (not advised)
+# or sslcontext=None to use those of the operating system
 
 
 
@@ -49,11 +58,11 @@
 FORWARD_OR_REVERSE = 'guess' # 'forward' (address -> coordinates) or 'reverse' (coordinates -> address)
                              # With 'guess' the script checks if the address is two numbers and then
                              # assumes reverse
-
+API_DOMAIN = 'api.opencagedata.com'
 MAX_ITEMS = 100              # How many lines to read from the input file. Set to 0 for unlimited
 NUM_WORKERS = 3              # For 10 requests per second try 2-5
 REQUEST_TIMEOUT_SECONDS = 5  # For individual HTTP requests. Default is 1
-RETRY_MAX_TRIES = 10         # How often to retry if a HTTP request times out
+RETRY_MAX_TRIES = 10          # How often to retry if a HTTP request times out
 RETRY_MAX_TIME = 60          # Limit in seconds for retries
 SHOW_PROGRESS = True         # Show progress bar
 
@@ -133,7 +142,7 @@ def backoff_hdlr(details):
                                             max_tries=RETRY_MAX_TRIES,
                                             on_backoff=backoff_hdlr)
 async def geocode_one_address(address, address_id):
-    async with OpenCageGeocode(API_KEY) as geocoder:
+    async with OpenCageGeocode(API_KEY, domain=API_DOMAIN, sslcontext=sslcontext) as geocoder:
         global FORWARD_OR_REVERSE
         try:
             if FORWARD_OR_REVERSE == 'reverse' or \
@@ -150,6 +159,9 @@ async def geocode_one_address(address, address_id):
                 # countrycode, language, etc
                 # see the full list: https://opencagedata.com/api#forward-opt
                 geocoding_results = await geocoder.geocode_async(address, no_annotations=1)
+        except SSLError as exc:
+            sys.stderr.write(str(exc))
+
         except Exception as exc:
             traceback.print_exception(exc, file=sys.stderr)
 

opencage/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = "OpenCage GmbH"
 __email__ = 'support@opencagedata.com'
-__version__ = '2.3.0'
+__version__ = '2.3.1'

opencage/geocoder.py

@@ -14,6 +14,8 @@
 except ImportError:
     AIOHTTP_AVAILABLE = False
 
+DEFAULT_DOMAIN = 'api.opencagedata.com'
+
 def backoff_max_time():
     return int(os.environ.get('BACKOFF_MAX_TIME', '120'))
 
@@ -102,6 +104,21 @@ class AioHttpError(OpenCageGeocodeError):
     """
 
 
+class SSLError(OpenCageGeocodeError):
+
+    """
+    Exception raised when SSL connection to OpenCage server fails.
+    """
+
+    def __unicode__(self):
+        """Convert exception to a string."""
+        return ("SSL Certificate error connecting to OpenCage API. This is usually due to "
+               "outdated CA root certificates of the operating system. "
+               )
+
+    __str__ = __unicode__
+
+
 class OpenCageGeocode:
 
     """
@@ -121,15 +138,18 @@ class OpenCageGeocode:
 
     """
 
-    url = 'https://api.opencagedata.com/geocode/v1/json'
-    key = ''
     session = None
 
-    def __init__(self, key, protocol='https'):
+    def __init__(self, key, protocol='https', domain=DEFAULT_DOMAIN, sslcontext=None):
         """Constructor."""
         self.key = key
-        if protocol and protocol == 'http':
-            self.url = self.url.replace('https://', 'http://')
+
+        if protocol and protocol not in ('http', 'https'):
+            protocol = 'https'
+        self.url = protocol + '://' + domain + '/geocode/v1/json'
+
+        # https://docs.aiohttp.org/en/stable/client_advanced.html#ssl-control-for-tcp-sockets
+        self.sslcontext = sslcontext
 
     def __enter__(self):
         self.session = requests.Session()
@@ -167,7 +187,7 @@ def geocode(self, query, **kwargs):
         """
 
         if self.session and isinstance(self.session, aiohttp.client.ClientSession):
-            raise AioHttpError("Cannot use `geocode` in an async context, use `gecode_async`.")
+            raise AioHttpError("Cannot use `geocode` in an async context, use `geocode_async`.")
 
         request = self._parse_request(query, kwargs)
         response = self._opencage_request(request)
@@ -271,34 +291,38 @@ def _opencage_request(self, params):
         return response_json
 
     async def _opencage_async_request(self, params):
-        async with self.session.get(self.url, params=params) as response:
-            try:
-                response_json = await response.json()
-            except ValueError as excinfo:
-                raise UnknownError("Non-JSON result from server") from excinfo
-
-            if response.status == 401:
-                raise NotAuthorizedError()
-
-            if response.status == 403:
-                raise ForbiddenError()
-
-            if response.status in (402, 429):
-                # Rate limit exceeded
-                print(response_json)
-                reset_time = datetime.utcfromtimestamp(response_json['rate']['reset'])
-                raise RateLimitExceededError(
-                    reset_to=int(response_json['rate']['limit']),
-                    reset_time=reset_time
-                )
-
-            if response.status == 500:
-                raise UnknownError("500 status code from API")
-
-            if 'results' not in response_json:
-                raise UnknownError("JSON from API doesn't have a 'results' key")
-
-            return response_json
+        try:
+            async with self.session.get(self.url, params=params, ssl=self.sslcontext) as response:
+                try:
+                    response_json = await response.json()
+                except ValueError as excinfo:
+                    raise UnknownError("Non-JSON result from server") from excinfo
+
+                if response.status == 401:
+                    raise NotAuthorizedError()
+
+                if response.status == 403:
+                    raise ForbiddenError()
+
+                if response.status in (402, 429):
+                    # Rate limit exceeded
+                    reset_time = datetime.utcfromtimestamp(response_json['rate']['reset'])
+                    raise RateLimitExceededError(
+                        reset_to=int(response_json['rate']['limit']),
+                        reset_time=reset_time
+                    )
+
+                if response.status == 500:
+                    raise UnknownError("500 status code from API")
+
+                if 'results' not in response_json:
+                    raise UnknownError("JSON from API doesn't have a 'results' key")
+
+                return response_json
+        except aiohttp.ClientSSLError as exp:
+            raise SSLError() from exp
+        except aiohttp.client_exceptions.ClientConnectorCertificateError as exp:
+            raise SSLError() from exp
 
     def _parse_request(self, query, params):
         if not isinstance(query, str):

setup.py

@@ -28,7 +28,7 @@
 
 setup(
     name="opencage",
-    version="2.3.0",
+    version="2.3.1",
     description="Wrapper module for the OpenCage Geocoder API",
     long_description=LONG_DESCRIPTION,
     long_description_content_type='text/markdown',

test/fixtures/badssl-com-chain.pem

@@ -0,0 +1,90 @@
+-----BEGIN CERTIFICATE-----
+MIIE8DCCA9igAwIBAgISA4mqZntfCH8MYyIVqUF1XZgpMA0GCSqGSIb3DQEBCwUA
+MDIxCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1MZXQncyBFbmNyeXB0MQswCQYDVQQD
+EwJSMzAeFw0yMzEwMTkxNTUwMjlaFw0yNDAxMTcxNTUwMjhaMBcxFTATBgNVBAMM
+DCouYmFkc3NsLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAONj
+dsqxZsR+pDzWX6GLCy6ImoAT60LNYvs9U6BIQ+fatIWbMELAFD6jY+IP25hrVEr1
+bgwRWmAAOnUc2qKXdtx6KXXO3cAJoCSHFNBDEZqzg/+exj+3emQH8dVZiYAS2Rpd
+nL9uKc3xgDDb74p1m7J4JdMewHmebRUmMt0MbA0f8sxvhbv9wIXkgAZd6dKYPGzJ
+KJlCoQifPiJ66JwYk8WVGEJH9m8LNDse388MscfsuwvAAh9tt2Fq6rmV9s21P6qf
+JgjePl65e8fVjsEWBAvC/aMYvTUs7Gdqej0qByjESpt1LZClNomJDvIgqA9+5KsU
+yCXigT6OiPjtZhdhgw0CAwEAAaOCAhkwggIVMA4GA1UdDwEB/wQEAwIFoDAdBgNV
+HSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4E
+FgQUqryJ2HM8bXniU+mPXLakkgUQobMwHwYDVR0jBBgwFoAUFC6zF7dYVsuuUAlA
+5h+vnYsUwsYwVQYIKwYBBQUHAQEESTBHMCEGCCsGAQUFBzABhhVodHRwOi8vcjMu
+by5sZW5jci5vcmcwIgYIKwYBBQUHMAKGFmh0dHA6Ly9yMy5pLmxlbmNyLm9yZy8w
+IwYDVR0RBBwwGoIMKi5iYWRzc2wuY29tggpiYWRzc2wuY29tMBMGA1UdIAQMMAow
+CAYGZ4EMAQIBMIIBAwYKKwYBBAHWeQIEAgSB9ASB8QDvAHUA2ra/az+1tiKfm8K7
+XGvocJFxbLtRhIU0vaQ9MEjX+6sAAAGLSNh8nwAABAMARjBEAiBkJnQowOqs+tDj
+7qXXu0PlDCvgvtEemuw1OvInlaHSrAIgcCZV5dJmGVrS1voinEpAzScJejhGB0vb
+G8dfKhJZD+wAdgA7U3d1Pi25gE6LMFsG/kA7Z9hPw/THvQANLXJv4frUFwAAAYtI
+2HyZAAAEAwBHMEUCIQCJ+gamX0P/hGiIuu70hn8d0svHSOAMJs3D+eOjMVqsywIg
+JXR/lAknUTRU+SyfySDoQ22bDSXfYWZGHLFgAkiRo48wDQYJKoZIhvcNAQELBQAD
+ggEBAGE3PDg7p2N8aZyAyO0pGVb/ob9opu12g+diNIdRSjsKIE+TO3uClM2OxT0t
+5GBz6Owbe010MQtqBKmX4Zm2LSLUm1kVhPh2ohWmA4hTyN3RG5W0IJ3red6VjrJY
+URhZQoXQb0gonxMs+zC+4GQ7+yqzWA1UkrWrURjjJCuljyoWF9sE7qEweomSQWnV
+v6bIF599/di1R2l5vcRq1DsQDgKaFY4IpKnvh3RhgO19YxlSS9ERRGBem3Aml9tb
+Yac12RmyuxsEAr0v75YeL3pAuq/1Rd5OeKfkm+K06Px3LxwcF92RljXkH6T2U8VM
+PEFKedHjYjAag3DUMqSuuGI+ONU=
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIFFjCCAv6gAwIBAgIRAJErCErPDBinU/bWLiWnX1owDQYJKoZIhvcNAQELBQAw
+TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
+cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMjAwOTA0MDAwMDAw
+WhcNMjUwOTE1MTYwMDAwWjAyMQswCQYDVQQGEwJVUzEWMBQGA1UEChMNTGV0J3Mg
+RW5jcnlwdDELMAkGA1UEAxMCUjMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
+AoIBAQC7AhUozPaglNMPEuyNVZLD+ILxmaZ6QoinXSaqtSu5xUyxr45r+XXIo9cP
+R5QUVTVXjJ6oojkZ9YI8QqlObvU7wy7bjcCwXPNZOOftz2nwWgsbvsCUJCWH+jdx
+sxPnHKzhm+/b5DtFUkWWqcFTzjTIUu61ru2P3mBw4qVUq7ZtDpelQDRrK9O8Zutm
+NHz6a4uPVymZ+DAXXbpyb/uBxa3Shlg9F8fnCbvxK/eG3MHacV3URuPMrSXBiLxg
+Z3Vms/EY96Jc5lP/Ooi2R6X/ExjqmAl3P51T+c8B5fWmcBcUr2Ok/5mzk53cU6cG
+/kiFHaFpriV1uxPMUgP17VGhi9sVAgMBAAGjggEIMIIBBDAOBgNVHQ8BAf8EBAMC
+AYYwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMBIGA1UdEwEB/wQIMAYB
+Af8CAQAwHQYDVR0OBBYEFBQusxe3WFbLrlAJQOYfr52LFMLGMB8GA1UdIwQYMBaA
+FHm0WeZ7tuXkAXOACIjIGlj26ZtuMDIGCCsGAQUFBwEBBCYwJDAiBggrBgEFBQcw
+AoYWaHR0cDovL3gxLmkubGVuY3Iub3JnLzAnBgNVHR8EIDAeMBygGqAYhhZodHRw
+Oi8veDEuYy5sZW5jci5vcmcvMCIGA1UdIAQbMBkwCAYGZ4EMAQIBMA0GCysGAQQB
+gt8TAQEBMA0GCSqGSIb3DQEBCwUAA4ICAQCFyk5HPqP3hUSFvNVneLKYY611TR6W
+PTNlclQtgaDqw+34IL9fzLdwALduO/ZelN7kIJ+m74uyA+eitRY8kc607TkC53wl
+ikfmZW4/RvTZ8M6UK+5UzhK8jCdLuMGYL6KvzXGRSgi3yLgjewQtCPkIVz6D2QQz
+CkcheAmCJ8MqyJu5zlzyZMjAvnnAT45tRAxekrsu94sQ4egdRCnbWSDtY7kh+BIm
+lJNXoB1lBMEKIq4QDUOXoRgffuDghje1WrG9ML+Hbisq/yFOGwXD9RiX8F6sw6W4
+avAuvDszue5L3sz85K+EC4Y/wFVDNvZo4TYXao6Z0f+lQKc0t8DQYzk1OXVu8rp2
+yJMC6alLbBfODALZvYH7n7do1AZls4I9d1P4jnkDrQoxB3UqQ9hVl3LEKQ73xF1O
+yK5GhDDX8oVfGKF5u+decIsH4YaTw7mP3GFxJSqv3+0lUFJoi5Lc5da149p90Ids
+hCExroL1+7mryIkXPeFM5TgO9r0rvZaBFOvV2z0gp35Z0+L4WPlbuEjN/lxPFin+
+HlUjr8gRsI3qfJOQFy/9rKIJR0Y/8Omwt/8oTWgy1mdeHmmjk7j1nYsvC9JSQ6Zv
+MldlTTKB3zhThV1+XWYp6rjd5JW1zbVWEkLNxE7GJThEUG3szgBVGP7pSWTUTsqX
+nLRbwHOoq7hHwg==
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIFYDCCBEigAwIBAgIQQAF3ITfU6UK47naqPGQKtzANBgkqhkiG9w0BAQsFADA/
+MSQwIgYDVQQKExtEaWdpdGFsIFNpZ25hdHVyZSBUcnVzdCBDby4xFzAVBgNVBAMT
+DkRTVCBSb290IENBIFgzMB4XDTIxMDEyMDE5MTQwM1oXDTI0MDkzMDE4MTQwM1ow
+TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
+cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwggIiMA0GCSqGSIb3DQEB
+AQUAA4ICDwAwggIKAoICAQCt6CRz9BQ385ueK1coHIe+3LffOJCMbjzmV6B493XC
+ov71am72AE8o295ohmxEk7axY/0UEmu/H9LqMZshftEzPLpI9d1537O4/xLxIZpL
+wYqGcWlKZmZsj348cL+tKSIG8+TA5oCu4kuPt5l+lAOf00eXfJlII1PoOK5PCm+D
+LtFJV4yAdLbaL9A4jXsDcCEbdfIwPPqPrt3aY6vrFk/CjhFLfs8L6P+1dy70sntK
+4EwSJQxwjQMpoOFTJOwT2e4ZvxCzSow/iaNhUd6shweU9GNx7C7ib1uYgeGJXDR5
+bHbvO5BieebbpJovJsXQEOEO3tkQjhb7t/eo98flAgeYjzYIlefiN5YNNnWe+w5y
+sR2bvAP5SQXYgd0FtCrWQemsAXaVCg/Y39W9Eh81LygXbNKYwagJZHduRze6zqxZ
+Xmidf3LWicUGQSk+WT7dJvUkyRGnWqNMQB9GoZm1pzpRboY7nn1ypxIFeFntPlF4
+FQsDj43QLwWyPntKHEtzBRL8xurgUBN8Q5N0s8p0544fAQjQMNRbcTa0B7rBMDBc
+SLeCO5imfWCKoqMpgsy6vYMEG6KDA0Gh1gXxG8K28Kh8hjtGqEgqiNx2mna/H2ql
+PRmP6zjzZN7IKw0KKP/32+IVQtQi0Cdd4Xn+GOdwiK1O5tmLOsbdJ1Fu/7xk9TND
+TwIDAQABo4IBRjCCAUIwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYw
+SwYIKwYBBQUHAQEEPzA9MDsGCCsGAQUFBzAChi9odHRwOi8vYXBwcy5pZGVudHJ1
+c3QuY29tL3Jvb3RzL2RzdHJvb3RjYXgzLnA3YzAfBgNVHSMEGDAWgBTEp7Gkeyxx
++tvhS5B1/8QVYIWJEDBUBgNVHSAETTBLMAgGBmeBDAECATA/BgsrBgEEAYLfEwEB
+ATAwMC4GCCsGAQUFBwIBFiJodHRwOi8vY3BzLnJvb3QteDEubGV0c2VuY3J5cHQu
+b3JnMDwGA1UdHwQ1MDMwMaAvoC2GK2h0dHA6Ly9jcmwuaWRlbnRydXN0LmNvbS9E
+U1RST09UQ0FYM0NSTC5jcmwwHQYDVR0OBBYEFHm0WeZ7tuXkAXOACIjIGlj26Ztu
+MA0GCSqGSIb3DQEBCwUAA4IBAQAKcwBslm7/DlLQrt2M51oGrS+o44+/yQoDFVDC
+5WxCu2+b9LRPwkSICHXM6webFGJueN7sJ7o5XPWioW5WlHAQU7G75K/QosMrAdSW
+9MUgNTP52GE24HGNtLi1qoJFlcDyqSMo59ahy2cI2qBDLKobkx/J3vWraV0T9VuG
+WCLKTVXkcGdtwlfFRjlBz4pYg1htmf5X6DYO8A4jqv2Il9DjXA6USbW1FzXSLr9O
+he8Y4IWS6wY7bCkjCWDcRQJMEhg76fsO3txE+FiYruq9RUWhiF1myv4Q6W+CyBFC
+Dfvp7OOGAN6dEOM4+qR9sdjoSYKEBpsr6GtPAQw4dy753ec5
+-----END CERTIFICATE-----

test/test_async.py

@@ -39,4 +39,4 @@ async def test_using_non_async_method():
         with pytest.raises(AioHttpError) as excinfo:
             await geocoder.geocode("Atlantis")
 
-    assert str(excinfo.value) == 'Cannot use `geocode` in an async context, use `gecode_async`.'
+    assert str(excinfo.value) == 'Cannot use `geocode` in an async context, use `geocode_async`.'

test/test_error_ssl.py

@@ -0,0 +1,27 @@
+# encoding: utf-8
+
+import ssl
+import pytest
+from opencage.geocoder import OpenCageGeocode, SSLError
+
+# NOTE: Testing keys https://opencagedata.com/api#testingkeys
+
+# Connect to a host that has an invalid certificate
+@pytest.mark.asyncio
+async def test_sslerror():
+    bad_domain = 'wrong.host.badssl.com'
+    async with OpenCageGeocode('6d0e711d72d74daeb2b0bfd2a5cdfdba', domain=bad_domain) as geocoder:
+        with pytest.raises(SSLError) as excinfo:
+            await geocoder.geocode_async("something")
+        assert str(excinfo.value).startswith('SSL Certificate error')
+
+# Connect to OpenCage API domain but use certificate of another domain
+# This tests that sslcontext can be set.
+@pytest.mark.asyncio
+async def test_sslerror_wrong_certificate():
+    sslcontext = ssl.create_default_context(cafile='test/fixtures/badssl-com-chain.pem')
+
+    async with OpenCageGeocode('6d0e711d72d74daeb2b0bfd2a5cdfdba', sslcontext=sslcontext) as geocoder:
+        with pytest.raises(SSLError) as excinfo:
+            await geocoder.geocode_async("something")
+        assert str(excinfo.value).startswith('SSL Certificate error')