Support registration in Consul and created a driver to connect other Services Discovery (#221)

* feat: added basic consul register

* refactor!: init actions in services, not in create_app

* chore: increment coverage tests

* feat: Added py-ms-consulate package, a fork from https://github.com/gmr/consulate

* tests: updated service discovery

Author: avara1986

.github/workflows/python-package.yml

@@ -36,7 +36,7 @@ jobs:
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
-        flake8 pyms --show-source --statistics --statistics
+        flake8 pyms --show-source --statistics
     - name: Lint with pylint
       run: |
         pylint --rcfile=pylintrc pyms

.pre-commit-config.yaml

@@ -0,0 +1,7 @@
+repos:
+  - repo: https://github.com/ambv/black
+    rev: stable
+    hooks:
+    - id: black
+      files: .
+

MANIFEST.in

@@ -4,5 +4,6 @@ include requirements-tests.txt
 recursive-include pyms *
 recursive-exclude tests *
 recursive-exclude examples *
+recursive-exclude docker *
 prune tests
 prune examples

Pipfile

@@ -4,7 +4,10 @@ url = "https://pypi.org/simple"
 verify_ssl = true
 
 [dev-packages]
-py-ms = {editable = true,extras = ["tests"], path = "."}
+py-ms = {editable = true,extras = ["tests"],path = "."}
 
 [packages]
-py-ms = {editable = true,extras = ["all"], path = "."}
+py-ms = {editable = true,extras = ["all"],path = "."}
+
+[pipenv]
+allow_prereleases = true

Pipfile.lock

@@ -1,5 +1,6 @@
 from examples.microservice_configuration import ms
+
 app = ms.create_app()
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     app.run()

examples/microservice_configuration/main.py

@@ -9,5 +9,5 @@ def example():
         "GLOBAL_VARIABLE": GLOBAL_VARIABLE,
         "GLOBAL_VARIABLE2": GLOBAL_VARIABLE2,
         "test1": config().test1,
-        "test2": config().test2
+        "test2": config().test2,
     }

examples/microservice_configuration/views.py

@@ -11,5 +11,5 @@ def example():
     return jsonify({"main": app.ms.config.encrypted_key})
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     app.run()

examples/microservice_crypt_aws_kms/main.py

@@ -1,4 +1,4 @@
-from flask import jsonify, current_app, request
+from flask import current_app, jsonify, request
 
 from pyms.flask.app import Microservice
 
@@ -12,5 +12,5 @@ def index():
     return jsonify({"main": "hello world {}".format(current_app.config["APP_NAME"])})
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     app.run()

examples/microservice_distribued_tracing/ms1/main.py

@@ -13,5 +13,5 @@ def index():
     return jsonify({"response": response.json()})
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     app.run(port=5001)

examples/microservice_distribued_tracing/ms2/main.py

@@ -1,3 +1,3 @@
 from pyms.flask.app import Microservice
 
-ms = Microservice()
+ms = Microservice()

examples/microservice_metrics/__init__.py

@@ -2,5 +2,5 @@
 
 app = ms.create_app()
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     app.run()

examples/microservice_metrics/main.py

@@ -7,4 +7,4 @@ def example():
     current_app.logger.info("start request")
     result = ms.requests.get_for_object("https://ghibliapi.herokuapp.com/films/2baf70d1-42bb-4437-b551-e5fed5a87abe")
     current_app.logger.info("end request")
-    return result
+    return result

examples/microservice_metrics/views.py

@@ -2,5 +2,5 @@
 
 app = ms.create_app()
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     app.run()

examples/microservice_requests/main.py

@@ -0,0 +1,11 @@
+pyms:
+  services:
+    service_discovery:
+      service: "examples.microservice_service_discovery.service.ServiceDiscoveryConsulBasic"
+      host: "localhost"
+      autoregister: true
+  config:
+    DEBUG: true
+    TESTING: false
+    APP_NAME: "Python Microservice My personal Service Discovery"
+    APPLICATION_ROOT: ""

examples/microservice_service_discovery/__init__.py

@@ -0,0 +1,15 @@
+from flask import jsonify
+
+from pyms.flask.app import Microservice
+
+ms = Microservice(path=__file__)
+app = ms.create_app()
+
+
+@app.route("/")
+def example():
+    return jsonify({"main": "hello world"})
+
+
+if __name__ == "__main__":
+    app.run()

examples/microservice_service_discovery/config.yml

@@ -0,0 +1,33 @@
+import json
+import uuid
+
+import requests
+
+from pyms.flask.services.service_discovery import ServiceDiscoveryBase
+
+
+class ServiceDiscoveryConsulBasic(ServiceDiscoveryBase):
+    id_app = str(uuid.uuid1())
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.host = config.host
+        self.port = config.port
+
+    def register_service(self, *args, **kwargs):
+        app_name = kwargs["app_name"]
+        healtcheck_url = kwargs["healtcheck_url"]
+        interval = kwargs["interval"]
+        headers = {"Content-Type": "application/json; charset=utf-8"}
+        data = {
+            "id": app_name + "-" + self.id_app,
+            "name": app_name,
+            "check": {"name": "ping check", "http": healtcheck_url, "interval": interval, "status": "passing"},
+        }
+        response = requests.put(
+            "http://{host}:{port}/v1/agent/service/register".format(host=self.host, port=self.port),
+            data=json.dumps(data),
+            headers=headers,
+        )
+        if response.status_code != 200:
+            raise Exception(response.content)

examples/microservice_service_discovery/main.py

@@ -0,0 +1,11 @@
+pyms:
+  services:
+    service_discovery:
+      service: "consul"
+      host: "localhost"
+      autoregister: true
+  config:
+    DEBUG: true
+    TESTING: false
+    APP_NAME: "Python Microservice"
+    APPLICATION_ROOT: ""

examples/microservice_service_discovery/service.py

@@ -0,0 +1,16 @@
+from flask import jsonify
+
+from pyms.flask.app import Microservice
+
+ms = Microservice(path=__file__)
+app = ms.create_app()
+
+
+@app.route("/")
+def example():
+    checks = ms.service_discovery.client.agent.checks()
+    return jsonify({"main": checks})
+
+
+if __name__ == "__main__":
+    app.run()

examples/microservice_service_discovery_consul/config.yml

@@ -3,5 +3,5 @@
 ms = Microservice()
 app = ms.create_app()
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     app.run()

examples/microservice_service_discovery_consul/main.py

@@ -1,5 +1,6 @@
 from examples.microservice_requests import ms
+
 app = ms.create_app()
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     app.run()

examples/microservice_swagger/main.py

@@ -11,5 +11,5 @@ def example():
     return jsonify({"main": "hello world"})
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     app.run()

examples/microservice_tracer/main.py

@@ -11,5 +11,5 @@ def example():
     return jsonify({"main": ms.config.environment})
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     app.run()

examples/mininum_microservice/main.py

@@ -12,7 +12,7 @@ def __init__(self, *args, **kwargs) -> None:
         self._init_boto()
         super().__init__(*args, **kwargs)
 
-    def encrypt(self, message: str) -> str:   # pragma: no cover
+    def encrypt(self, message: str) -> str:  # pragma: no cover
         ciphertext = self.client.encrypt(
             KeyId=self.config.key_id,
             Plaintext=bytes(message, encoding="UTF-8"),
@@ -22,16 +22,14 @@ def encrypt(self, message: str) -> str:   # pragma: no cover
     def _init_boto(self) -> None:  # pragma: no cover
         check_package_exists("boto3")
         boto3 = import_package("boto3")
-        boto3.set_stream_logger(name='botocore')
-        self.client = boto3.client('kms')
+        boto3.set_stream_logger(name="botocore")
+        self.client = boto3.client("kms")
 
     def _aws_decrypt(self, blob_text: bytes) -> str:  # pragma: no cover
         response = self.client.decrypt(
-            CiphertextBlob=blob_text,
-            KeyId=self.config.key_id,
-            EncryptionAlgorithm=self.encryption_algorithm
+            CiphertextBlob=blob_text, KeyId=self.config.key_id, EncryptionAlgorithm=self.encryption_algorithm
         )
-        return str(response['Plaintext'], encoding="UTF-8")
+        return str(response["Plaintext"], encoding="UTF-8")
 
     def _parse_encrypted(self, encrypted: str) -> bytes:
         blob_text = base64.b64decode(encrypted)

examples/mininum_microservice_docker/main.py

@@ -1,3 +1,3 @@
 from .main import Command
 
-__all__ = ['Command']
+__all__ = ["Command"]

pyms/cloud/aws/kms.py

@@ -1,16 +1,15 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals, print_function
+from __future__ import print_function, unicode_literals
 
 import argparse
 import os
 import sys
 from distutils.util import strtobool
 
+from pyms.config import create_conf_file
 from pyms.crypt.fernet import Crypt
 from pyms.flask.services.swagger import merge_swagger_file
 from pyms.utils import check_package_exists, import_from, utils
-from pyms.config import create_conf_file
 
 
 class Command:
@@ -27,37 +26,40 @@ def __init__(self, *args, **kwargs):
         if not arguments:  # pragma: no cover
             arguments = sys.argv[1:]
 
-        parser = argparse.ArgumentParser(description='Python Microservices')
+        parser = argparse.ArgumentParser(description="Python Microservices")
 
-        commands = parser.add_subparsers(title="Commands", description='Available commands', dest='command_name')
+        commands = parser.add_subparsers(title="Commands", description="Available commands", dest="command_name")
 
-        parser_encrypt = commands.add_parser('encrypt', help='Encrypt a string')
-        parser_encrypt.add_argument("encrypt", default='', type=str, help='Encrypt a string')
+        parser_encrypt = commands.add_parser("encrypt", help="Encrypt a string")
+        parser_encrypt.add_argument("encrypt", default="", type=str, help="Encrypt a string")
 
-        parser_create_key = commands.add_parser('create-key', help='Generate a Key to encrypt strings in config')
-        parser_create_key.add_argument("create_key", action='store_true',
-                                       help='Generate a Key to encrypt strings in config')
+        parser_create_key = commands.add_parser("create-key", help="Generate a Key to encrypt strings in config")
+        parser_create_key.add_argument(
+            "create_key", action="store_true", help="Generate a Key to encrypt strings in config"
+        )
 
         parser_startproject = commands.add_parser(
-            'startproject',
-            help='Generate a project from https://github.com/python-microservices/microservices-template')
+            "startproject",
+            help="Generate a project from https://github.com/python-microservices/microservices-template",
+        )
         parser_startproject.add_argument(
-            "startproject", action='store_true',
-            help='Generate a project from https://github.com/python-microservices/microservices-template')
+            "startproject",
+            action="store_true",
+            help="Generate a project from https://github.com/python-microservices/microservices-template",
+        )
 
         parser_startproject.add_argument(
-            "-b", "--branch",
-            help='Select a branch from https://github.com/python-microservices/microservices-template')
+            "-b", "--branch", help="Select a branch from https://github.com/python-microservices/microservices-template"
+        )
 
-        parser_merge_swagger = commands.add_parser('merge-swagger', help='Merge swagger into a single file')
-        parser_merge_swagger.add_argument("merge_swagger", action='store_true',
-                                          help='Merge swagger into a single file')
+        parser_merge_swagger = commands.add_parser("merge-swagger", help="Merge swagger into a single file")
+        parser_merge_swagger.add_argument("merge_swagger", action="store_true", help="Merge swagger into a single file")
         parser_merge_swagger.add_argument(
-            "-f", "--file", default=os.path.join('project', 'swagger', 'swagger.yaml'),
-            help='Swagger file path')
+            "-f", "--file", default=os.path.join("project", "swagger", "swagger.yaml"), help="Swagger file path"
+        )
 
-        parser_create_config = commands.add_parser('create-config', help='Generate a config file')
-        parser_create_config.add_argument("create_config", action='store_true', help='Generate a config file')
+        parser_create_config = commands.add_parser("create-config", help="Generate a config file")
+        parser_create_config.add_argument("create_config", action="store_true", help="Generate a config file")
 
         parser.add_argument("-v", "--verbose", default="", type=str, help="Verbose ")
 
@@ -100,10 +102,10 @@ def run(self):
         crypt = Crypt()
         if self.create_key:
             path = crypt._loader.get_path_from_env()  # pylint: disable=protected-access
-            pwd = self.get_input('Type a password to generate the key file: ')
+            pwd = self.get_input("Type a password to generate the key file: ")
             # Should use yes_no_input insted of get input below
             # the result should be validated for Yes (Y|y) rather allowing anything other than 'n'
-            generate_file = self.get_input('Do you want to generate a file in {}? [Y/n]'.format(path))
+            generate_file = self.get_input("Do you want to generate a file in {}? [Y/n]".format(path))
             generate_file = generate_file.lower() != "n"
             key = crypt.generate_key(pwd, generate_file)
             if generate_file:
@@ -118,7 +120,7 @@ def run(self):
         if self.startproject:
             check_package_exists("cookiecutter")
             cookiecutter = import_from("cookiecutter.main", "cookiecutter")
-            cookiecutter('gh:python-microservices/cookiecutter-pyms', checkout=self.branch)
+            cookiecutter("gh:python-microservices/cookiecutter-pyms", checkout=self.branch)
             self.print_ok("Created project OK")
         if self.merge_swagger:
             try:
@@ -128,8 +130,8 @@ def run(self):
                 self.print_error(ex.__str__())
                 return False
         if self.create_config:
-            use_requests = self.yes_no_input('Do you want to use request')
-            use_swagger = self.yes_no_input('Do you want to use swagger')
+            use_requests = self.yes_no_input("Do you want to use request")
+            use_swagger = self.yes_no_input("Do you want to use swagger")
             try:
                 conf_file_path = create_conf_file(use_requests, use_swagger)
                 self.print_ok(f'Config file "{conf_file_path}" created')
@@ -140,7 +142,9 @@ def run(self):
         return True
 
     def yes_no_input(self, msg=""):  # pragma: no cover
-        answer = input(utils.colored_text(f'{msg}{"?" if not msg.endswith("?") else ""} [Y/n] :', utils.Colors.BLUE, True))  # nosec
+        answer = input(  # nosec
+            utils.colored_text(f'{msg}{"?" if not msg.endswith("?") else ""} [Y/n] :', utils.Colors.BLUE, True)
+        )
         try:
             return strtobool(answer)
         except ValueError:
@@ -168,6 +172,6 @@ def exit_ok(self, msg=""):  # pragma: no cover
         sys.exit(0)
 
 
-if __name__ == '__main__':  # pragma: no cover
+if __name__ == "__main__":  # pragma: no cover
     cmd = Command(arguments=sys.argv[1:], autorun=False)
     cmd.run()