Maintenance of project and LocalStack configurations

Author: pinzon

.gitignore

@@ -1,3 +1,6 @@
 .venv
+.idea
 nosetests.xml
-__pycache__
+__pycache__
+localstack_utils.egg-info
+.ruff_cache

Makefile

@@ -1,21 +1,26 @@
+VENV_BIN ?= python3 -m venv
 VENV_DIR ?= .venv
-TEST_PATH ?= ./tests
-PIP_CMD ?= pip
-NOSE_LOG_LEVEL ?= WARNING
+PIP_CMD ?= pip3
 
 ifeq ($(OS), Windows_NT)
-	VENV_RUN = . $(VENV_DIR)/Scripts/activate
+	VENV_ACTIVATE = $(VENV_DIR)/Scripts/activate
 else
-	VENV_RUN = . $(VENV_DIR)/bin/activate
+	VENV_ACTIVATE = $(VENV_DIR)/bin/activate
 endif
 
-setup-venv:
-	(test `which virtualenv` || $(PIP_CMD) install --user virtualenv) && \
-		(test -e $(VENV_DIR) || virtualenv $(VENV_OPTS) $(VENV_DIR))
+VENV_RUN = . $(VENV_ACTIVATE)
 
-install-venv:
-	make setup-venv && \
-		test ! -e requirements.txt || ($(VENV_RUN); $(PIP_CMD) -q install -r requirements.txt)
+venv: $(VENV_ACTIVATE)
+
+format: venv           		  ## Run ruff and black to format the whole codebase
+	($(VENV_RUN); python -m ruff check --show-source --fix .; python -m black .)
+
+lint: venv      		  ## Run code linter to check code style and check if formatter would make changes
+	($(VENV_RUN); python -m ruff check --show-source . && python -m black --check .)
+
+install: venv
+	$(VENV_RUN); $(PIP_CMD) install -e .
+
+test: venv              	  ## Run tests
+	($(VENV_RUN); python -m pytest -v --cov=plux --cov-report=term-missing --cov-report=xml tests)
 
-test:
-	($(VENV_RUN); DEBUG=$(DEBUG) PYTHONPATH=`pwd` nosetests $(NOSE_ARGS) --with-timer --logging-level=$(NOSE_LOG_LEVEL) --nocapture --no-skip --exe --cover-erase --cover-tests --cover-inclusive --cover-package=localstack --with-xunit --exclude='$(VENV_DIR).*' --ignore-files='lambda_python3.py' $(TEST_PATH))

localstack_utils/__init__.py

@@ -3,54 +3,73 @@
 import docker
 from time import sleep
 
-LOCALSTACK_NAME = "localstack/localstack";
-LOCALSTACK_TAG = "latest";
-LOCALSTACK_PORT_EDGE = "4566";
-LOCALSTACK_PORT_ELASTICSEARCH = "4571";
-
-MAX_PORT_CONNECTION_ATTEMPTS = 10;
-MAX_LOG_COLLECTION_ATTEMPTS = 120;
-POLL_INTERVAL = 1;
-NUM_LOG_LINES = 10;
-
-ENV_DEBUG = "DEBUG";
-ENV_USE_SSL = "USE_SSL";
-ENV_DEBUG_DEFAULT = "1";
+LOCALSTACK_COMMUNITY_IMAGE_NAME = "localstack/localstack"
+LOCALSTACK_PRO_IMAGE_NAME = "localstack/localstack-pro"
+LATEST_TAG = "latest"
+
+MAX_PORT_CONNECTION_ATTEMPTS = 10
+MAX_LOG_COLLECTION_ATTEMPTS = 120
+POLL_INTERVAL = 1
+NUM_LOG_LINES = 10
+
+ENV_DEBUG = "DEBUG"
+ENV_USE_SSL = "USE_SSL"
+ENV_DEBUG_DEFAULT = "1"
 LOCALSTACK_EXTERNAL_HOSTNAME = "HOSTNAME_EXTERNAL"
-DEFAULT_CONTAINER_ID = "localstack_main"
+DEFAULT_CONTAINER_ID = "localstack-main"
 
 DOCKER_CLIENT = docker.from_env()
 
-class Container:
 
+class Container:
     @staticmethod
-    def create_localstack_container(external_hostname, pull_new_image, randomize_ports, image_name, image_tag, port_edge, port_elasticsearch, environment_variables, port_mappings, bind_mounts, platform):
+    def create_localstack_container(
+        pull_new_image: bool,
+        image_name: str = None,
+        image_tag: str = LATEST_TAG,
+        gateway_listen: str = "0.0.0.0:4566",
+        environment_variables: dict = None,
+        bind_ports: dict = None,
+        pro: bool = False,
+    ):
+        environment_variables = environment_variables or {}
+        environment_variables["GATEWAY_LISTEN"] = gateway_listen
+
+        image_name_or_default = image_name or (
+            LOCALSTACK_PRO_IMAGE_NAME if pro else LOCALSTACK_COMMUNITY_IMAGE_NAME
+        )
+        image_exists = (
+            True
+            if len(DOCKER_CLIENT.images.list(name=image_name_or_default))
+            else False
+        )
 
-        environment_variables = {} if environment_variables == None else environment_variables
-        bind_mounts = {} if bind_mounts == None else bind_mounts
-        port_mappings = {} if port_mappings == None else port_mappings
-        image_name_or_default = LOCALSTACK_NAME if image_name == None else image_name 
-        image_exists = True if len(DOCKER_CLIENT.images.list(name=image_name_or_default)) else False
-        
-        fullPortEdge = {(LOCALSTACK_PORT_EDGE if port_edge == None else port_edge) : (LOCALSTACK_PORT_EDGE)}
+        bind_ports = bind_ports or {}
+        gateway_port = gateway_listen.split(":")[1]
+        bind_ports.update({gateway_port: gateway_port})
 
         if pull_new_image or not image_exists:
             logging.info("Pulling latest image")
             DOCKER_CLIENT.images.pull(image_name_or_default, image_tag)
-        
-        return DOCKER_CLIENT.containers.run(image_name_or_default, ports=fullPortEdge, environment=environment_variables, detach=True)
-    
+
+        return DOCKER_CLIENT.containers.run(
+            image_name_or_default,
+            ports=bind_ports,
+            environment=environment_variables,
+            detach=True,
+        )
+
     @staticmethod
-    def waitForReady(container, pattern):
-        attemps = 0
-        
+    def wait_for_ready(container, pattern):
+        attempts = 0
+
         while True:
-            logs = container.logs(tail=NUM_LOG_LINES).decode('utf-8')
+            logs = container.logs(tail=NUM_LOG_LINES).decode("utf-8")
             if re.search(pattern, logs):
-                return;
-            
+                return
+
             sleep(POLL_INTERVAL)
-            attemps += 1
+            attempts += 1
 
-            if attemps >= MAX_LOG_COLLECTION_ATTEMPTS:
-                raise "Could not find token: " +pattern.toString()+ "in logs"
+            if attempts >= MAX_LOG_COLLECTION_ATTEMPTS:
+                raise "Could not find token: " + pattern.toString() + "in logs"

localstack_utils/container.py

@@ -3,7 +3,9 @@
 import docker
 import logging
 from localstack_utils.container import Container
-from localstack_utils.localstack_docker_configuration import LocalstackDockerConfiguration
+from localstack_utils.localstack_docker_configuration import (
+    LocalstackDockerConfiguration,
+)
 from localstack_utils.localstack_logger import LocalstackLogger
 
 ENV_CONFIG_USE_SSL = "USE_SSL"
@@ -12,14 +14,16 @@
 TMP_PATH = "/tmp/localstack"
 READY_TOKEN = re.compile("Ready\\.")
 DEFAULT_EDGE_PORT = 4566
-PORT_CONFIG_FILENAME = "/opt/code/localstack/.venv/lib/python3.8/site-packages/localstack_client/config.py"
+PORT_CONFIG_FILENAME = (
+    "/opt/code/localstack/.venv/lib/python3.8/site-packages/localstack_client/config.py"
+)
 # DEFAULT_PORT_PATTERN = re.compile("'(\\w+)'\\Q: '{proto}://{host}:\\E(\\d+)'")
 
 localstack_instance = None
-logging.basicConfig(level=logging.INFO, format='%(message)s')
+logging.basicConfig(level=logging.INFO, format="%(message)s")
+
 
 class Localstack:
-    
     localstack_container = None
     service_to_map = {}
     locked = False
@@ -28,62 +32,78 @@ class Localstack:
     def INSTANCE():
         return Localstack()
 
-    external_hostName = '' 
+    external_hostName = ""
 
-    def startup(self, docker_configuration) :
+    def startup(self, docker_configuration):
         if self.locked:
             raise Exception("A docker instance is starting or already started.")
         self.locked = True
-        self.external_hostname = docker_configuration.external_hostname
 
         try:
             self.localstack_container = Container.create_localstack_container(
-                docker_configuration.external_hostname,
                 docker_configuration.pull_new_image,
-                docker_configuration.randomize_ports,
                 docker_configuration.image_name,
                 docker_configuration.image_tag,
-                docker_configuration.port_edge,
-                docker_configuration.port_elastic_search,
+                docker_configuration.gateway_listen,
                 docker_configuration.environment_variables,
                 docker_configuration.port_mappings,
-                docker_configuration.bind_mounts,
-                docker_configuration.platform
+                docker_configuration.pro,
             )
 
             self.setup_logger()
 
-            Container.waitForReady(self.localstack_container, READY_TOKEN)
+            Container.wait_for_ready(self.localstack_container, READY_TOKEN)
 
-        except docker.errors.APIError as error:
+        except docker.errors.APIError:
             if not docker_configuration.ignore_docker_runerrors:
                 raise "Unable to start docker"
 
-        except:
+        except Exception:
             raise sys.exc_info()
 
-
     def stop(self):
         self.localstack_container.stop()
 
-    
     def setup_logger(self):
         localstack_logger = LocalstackLogger(self.localstack_container)
         localstack_logger.start()
 
-def startup_localstack(port=4566, services=[], ignore_docker_errors = False):
+
+def startup_localstack(
+    image_name="",
+    tag="",
+    pro=False,
+    ports=None,
+    env_variables=None,
+    gateway_listen="",
+    ignore_docker_errors=False,
+):
     global localstack_instance
     localstack_instance = Localstack.INSTANCE()
     config = LocalstackDockerConfiguration()
 
-    if len(services):
-        config.envirement_variables.update({'SERVICES': ','.join(services)})
-    if port != 4566:
-        config.port_edge = str(port)
     config.ignore_docker_runerrors = ignore_docker_errors
+    if image_name:
+        config.image_name = image_name
+
+    if tag:
+        config.image_tag = tag
+
+    if ports:
+        config.port_mappings = ports
+
+    if pro:
+        config.pro = pro
+
+    if env_variables:
+        config.environment_variables = env_variables
+
+    if gateway_listen:
+        config.gateway_listen = gateway_listen
 
     localstack_instance.startup(config)
 
+
 def stop_localstack():
     if localstack_instance:
         localstack_instance.stop()

localstack_utils/localstack_utils.py localstack_utils/localstack.pylocalstack_utils/localstack_utils.py renamed to localstack_utils/localstack.py

@@ -5,13 +5,12 @@ class LocalstackDockerConfiguration:
     image_tag = None
     platform = None
 
-    port_edge = '4566'
-    port_elastic_search = '4571'
-    external_hostname = 'localhost'
+    gateway_listen = "0.0.0.0:4566"
     environment_variables = {}
     port_mappings = {}
     bind_mounts = {}
 
     initialization_token = None
     use_dingle_docker_container = False
     ignore_docker_runerrors = False
+    pro = False

localstack_utils/localstack_docker_configuration.py

@@ -1,19 +1,24 @@
 import threading
 import logging
 
-class LocalstackLogger():
+
+class LocalstackLogger:
     stream = None
     log_thread = None
+
     def __init__(self, localsack_container) -> None:
-        self.stream = localsack_container.logs(stream=True, follow=True)    
-    
+        self.stream = localsack_container.logs(stream=True, follow=True)
+
     def start(self):
-        self.log_thread = threading.Thread(target=log_stream,args=[self.stream]).start()
-            
+        self.log_thread = threading.Thread(
+            target=log_stream, args=[self.stream]
+        ).start()
+
+
 def log_stream(stream):
     try:
         while True:
-            line = next(stream).decode("utf-8").replace("\n",'')
+            line = next(stream).decode("utf-8").replace("\n", "")
             logging.info(line)
     except StopIteration:
         pass

localstack_utils/localstack_logger.py

@@ -0,0 +1,49 @@
+[metadata]
+name = localstack-utils
+version = attr: localstack_utils.__version__
+author = Cristopher Pinzon
+author_email = cristopher.pinzon@localstack.cloud
+summary = LocalStack Utils for testing
+description = Utils for testing with LocalStack
+long_description = file: README.md
+long_description_content_type = text/markdown; charset=UTF-8
+license = Apache License 2.0
+classifiers =
+    Development Status :: 5 - Production/Stable
+    License :: OSI Approved :: Apache Software License
+    Operating System :: OS Independent
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
+    Programming Language :: Python :: 3.11
+    Topic :: Software Development :: Libraries
+    Topic :: Utilities
+
+[options]
+zip_safe = False
+packages = find:
+install_requires =
+    docker>=6.1.1
+    nose>=1.3.7
+    nose-timer>=0.7.5
+test_requires =
+    pytest>=7.4.0
+    boto3>=1.26.121
+
+[options.extras_require]
+dev =
+    boto3>=1.26.121
+    pytest>=7.4.0
+    black==23.10.0
+    ruff==0.1.0
+
+[options.packages.find]
+exclude =
+    tests*
+
+[options.package_data]
+* = *.md
+exclude =
+    tests
+    tests.*