Spaces:
Paused
Paused
| if __name__ == '__main__': | |
| import sys | |
| print("Critical - GAD must be started as a python module, for example using python -m gitautodeploy") | |
| sys.exit() | |
| class LogInterface(object): | |
| """Interface that functions as a stdout and stderr handler and directs the | |
| output to the logging module, which in turn will output to either console, | |
| file or both.""" | |
| def __init__(self, level=None): | |
| import logging | |
| self.level = (level if level else logging.getLogger().info) | |
| def write(self, msg): | |
| for line in msg.strip().split("\n"): | |
| self.level(line) | |
| def flush(self): | |
| pass | |
| from .wsserver import WebSocketClientHandlerFactory | |
| from .httpserver import WebhookRequestHandlerFactory | |
| class GitAutoDeploy(object): | |
| _instance = None | |
| _http_server = None | |
| _https_server = None | |
| _https_server_unwrapped_socket = None | |
| _config = {} | |
| _server_status = {} | |
| _pid = None | |
| _event_store = None | |
| _default_stdout = None | |
| _default_stderr = None | |
| _startup_event = None | |
| _ws_clients = [] | |
| _http_port = None | |
| def __new__(cls, *args, **kwargs): | |
| """Overload constructor to enable singleton access""" | |
| if not cls._instance: | |
| cls._instance = super(GitAutoDeploy, cls).__new__( | |
| cls, *args, **kwargs) | |
| return cls._instance | |
| def __init__(self): | |
| from .events import EventStore, StartupEvent | |
| # Setup an event store instance that can keep a global record of events | |
| self._event_store = EventStore() | |
| self._event_store.register_observer(self) | |
| # Create a startup event that can hold status and any error messages | |
| # from the startup process | |
| self._startup_event = StartupEvent() | |
| self._event_store.register_action(self._startup_event) | |
| def clone_all_repos(self): | |
| """Iterates over all configured repositories and clones them to their | |
| configured paths.""" | |
| import os | |
| import re | |
| import logging | |
| from .wrappers import GitWrapper | |
| logger = logging.getLogger() | |
| if 'repositories' not in self._config: | |
| return | |
| # Iterate over all configured repositories | |
| for repo_config in self._config['repositories']: | |
| # Only clone repositories with a configured path | |
| if 'url' not in repo_config: | |
| logger.critical("Repository has no configured URL") | |
| self.exit() | |
| return | |
| # Only clone repositories with a configured path | |
| if 'path' not in repo_config: | |
| logger.debug("Repository %s will not be cloned (no path configured)" % repo_config['url']) | |
| continue | |
| if os.path.isdir(repo_config['path']) and os.path.isdir(repo_config['path']+'/.git'): | |
| GitWrapper.init(repo_config) | |
| else: | |
| GitWrapper.clone(repo_config) | |
| def ssh_key_scan(self): | |
| import re | |
| import logging | |
| from .wrappers import ProcessWrapper | |
| logger = logging.getLogger() | |
| for repository in self._config['repositories']: | |
| if 'url' not in repository: | |
| continue | |
| logger.info("Scanning repository: %s" % repository['url']) | |
| m = re.match('[^\@]+\@([^\:\/]+)(:(\d+))?', repository['url']) | |
| if m is not None: | |
| host = m.group(1) | |
| port = m.group(3) | |
| port_arg = '' if port is None else ('-p %s ' % port) | |
| cmd = 'ssh-keyscan %s%s >> $HOME/.ssh/known_hosts' % (port_arg, host) | |
| ProcessWrapper().call([cmd], shell=True) | |
| else: | |
| logger.error('Could not find regexp match in path: %s' % repository['url']) | |
| def create_pid_file(self): | |
| import os | |
| with open(self._config['pid-file'], 'w') as f: | |
| f.write(str(os.getpid())) | |
| def read_pid_file(self): | |
| with open(self._config['pid-file'], 'r') as f: | |
| return f.readlines() | |
| def remove_pid_file(self): | |
| import os | |
| import errno | |
| if 'pid-file' in self._config and self._config['pid-file']: | |
| try: | |
| os.remove(self._config['pid-file']) | |
| except OSError as e: | |
| # errno.ENOENT = no such file or directory | |
| if e.errno != errno.ENOENT: | |
| raise | |
| def create_daemon(): | |
| import os | |
| try: | |
| # Spawn first child. Returns 0 in the child and pid in the parent. | |
| pid = os.fork() | |
| except OSError as e: | |
| raise Exception("%s [%d]" % (e.strerror, e.errno)) | |
| # First child | |
| if pid == 0: | |
| os.setsid() | |
| try: | |
| # Spawn second child | |
| pid = os.fork() | |
| except OSError as e: | |
| raise Exception("%s [%d]" % (e.strerror, e.errno)) | |
| if pid == 0: | |
| os.umask(0) | |
| else: | |
| # Kill first child | |
| os._exit(0) | |
| else: | |
| # Kill parent of first child | |
| os._exit(0) | |
| return 0 | |
| def update(self, *args, **kwargs): | |
| import json | |
| data = json.dumps(kwargs).encode('utf-8') | |
| for client in self._ws_clients: | |
| client.sendMessage(data) | |
| def get_log_formatter(self): | |
| import logging | |
| return logging.Formatter("%(asctime)s [%(levelname)-5.5s] %(message)s") | |
| def setup_console_logger(self): | |
| import logging | |
| # Set up logging | |
| logger = logging.getLogger() | |
| consoleHandler = logging.StreamHandler() | |
| consoleHandler.setFormatter(self.get_log_formatter()) | |
| # Check if a stream handler is already present (will be if GAD is started by test script) | |
| handler_present = False | |
| for handler in logger.handlers: | |
| if isinstance(handler, type(consoleHandler)): | |
| handler_present = True | |
| break | |
| if not handler_present: | |
| logger.addHandler(consoleHandler) | |
| def setup(self, config): | |
| """Setup an instance of GAD based on the provided config object.""" | |
| import sys | |
| import socket | |
| import os | |
| import logging | |
| import base64 | |
| from .lock import Lock | |
| import getpass | |
| # This solves https://github.com/olipo186/Git-Auto-Deploy/issues/118 | |
| try: | |
| from logging import NullHandler | |
| except ImportError: | |
| from logging import Handler | |
| class NullHandler(Handler): | |
| def emit(self, record): | |
| pass | |
| # Attatch config values to this instance | |
| self._config = config | |
| # Set up logging | |
| logger = logging.getLogger() | |
| logFormatter = self.get_log_formatter() | |
| # Enable console output? | |
| if ('quiet' in self._config and self._config['quiet']) or ('daemon-mode' in self._config and self._config['daemon-mode']): | |
| # Add a default null handler that suppresses any console output | |
| logger.addHandler(NullHandler()) | |
| else: | |
| # Set up console logger if not already present | |
| self.setup_console_logger() | |
| # Set logging level | |
| if 'log-level' in self._config: | |
| level = logging.getLevelName(self._config['log-level']) | |
| logger.setLevel(level) | |
| if 'log-file' in self._config and self._config['log-file']: | |
| # Translate any ~ in the path into /home/<user> | |
| fileHandler = logging.FileHandler(self._config['log-file']) | |
| fileHandler.setFormatter(logFormatter) | |
| logger.addHandler(fileHandler) | |
| # Display a warning when trying to run as root | |
| if not self._config['allow-root-user'] and getpass.getuser() == 'root': | |
| logger.critical("Refusing to start as root. This application shouldn't run as a privileged used. Please run it as a different user. To disregard this warning and start anyway, set the config option \"allow-root-user\" to true, or use the command line argument --allow-root-user") | |
| sys.exit() | |
| if 'ssh-keyscan' in self._config and self._config['ssh-keyscan']: | |
| self._startup_event.log_info('Scanning repository hosts for ssh keys...') | |
| self.ssh_key_scan() | |
| # Clone all repos once initially | |
| self.clone_all_repos() | |
| # Set default stdout and stderr to our logging interface (that writes | |
| # to file and console depending on user preference) | |
| if 'intercept-stdout' in self._config and self._config['intercept-stdout']: | |
| self._default_stdout = sys.stdout | |
| self._default_stderr = sys.stderr | |
| sys.stdout = LogInterface(logger.info) | |
| sys.stderr = LogInterface(logger.error) | |
| if 'daemon-mode' in self._config and self._config['daemon-mode']: | |
| self._startup_event.log_info('Starting Git Auto Deploy in daemon mode') | |
| GitAutoDeploy.create_daemon() | |
| self._pid = os.getpid() | |
| self.create_pid_file() | |
| # Generate auth key to protect the web socket server | |
| self._server_status['auth-key'] = base64.b64encode(os.urandom(32)) | |
| # Clear any existing lock files, with no regard to possible ongoing processes | |
| for repo_config in self._config['repositories']: | |
| # Do we have a physical repository? | |
| if 'path' in repo_config: | |
| Lock(os.path.join(repo_config['path'], 'status_running')).clear() | |
| Lock(os.path.join(repo_config['path'], 'status_waiting')).clear() | |
| #if 'daemon-mode' not in self._config or not self._config['daemon-mode']: | |
| # self._startup_event.log_info('Git Auto Deploy started') | |
| def serve_http(self, serve_forever=True): | |
| """Starts a HTTP server that listens for webhook requests and serves the web ui.""" | |
| import sys | |
| import socket | |
| import os | |
| from .events import SystemEvent | |
| try: | |
| from BaseHTTPServer import HTTPServer | |
| except ImportError as e: | |
| from http.server import HTTPServer | |
| if not self._config['http-enabled']: | |
| return | |
| # Setup | |
| try: | |
| # Create web hook request handler class | |
| WebhookRequestHandler = WebhookRequestHandlerFactory(self._config, self._event_store, self._server_status, is_https=False) | |
| # Create HTTP server | |
| self._http_server = HTTPServer((self._config['http-host'], | |
| self._config['http-port']), | |
| WebhookRequestHandler) | |
| # Setup SSL for HTTP server | |
| sa = self._http_server.socket.getsockname() | |
| self._http_port = sa[1] | |
| self._server_status['http-uri'] = "http://%s:%s" % (self._config['http-host'], sa[1]) | |
| self._startup_event.log_info("Listening for connections on %s" % self._server_status['http-uri']) | |
| self._startup_event.http_address = sa[0] | |
| self._startup_event.http_port = sa[1] | |
| self._startup_event.set_http_started(True) | |
| except socket.error as e: | |
| self._startup_event.log_critical("Unable to start HTTP server: %s" % e) | |
| return | |
| if not serve_forever: | |
| return | |
| # Run forever | |
| try: | |
| self._http_server.serve_forever() | |
| except socket.error as e: | |
| event = SystemEvent() | |
| self._event_store.register_action(event) | |
| event.log_critical("Error on socket: %s" % e) | |
| sys.exit(1) | |
| except KeyboardInterrupt as e: | |
| event = SystemEvent() | |
| self._event_store.register_action(event) | |
| event.log_info('Requested close by keyboard interrupt signal') | |
| self.stop() | |
| self.exit() | |
| event = SystemEvent() | |
| self._event_store.register_action(event) | |
| event.log_info('HTTP server did quit') | |
| def serve_https(self): | |
| """Starts a HTTPS server that listens for webhook requests and serves the web ui.""" | |
| import sys | |
| import socket | |
| import os | |
| import ssl | |
| from .events import SystemEvent | |
| try: | |
| from BaseHTTPServer import HTTPServer | |
| except ImportError as e: | |
| from http.server import HTTPServer | |
| if not self._config['https-enabled']: | |
| return | |
| if not os.path.isfile(self._config['ssl-cert']): | |
| self._startup_event.log_critical("Unable to activate SSL: File does not exist: %s" % self._config['ssl-cert']) | |
| return | |
| # Setup | |
| try: | |
| # Create web hook request handler class | |
| WebhookRequestHandler = WebhookRequestHandlerFactory(self._config, self._event_store, self._server_status, is_https=True) | |
| # Create HTTP server | |
| self._https_server = HTTPServer((self._config['https-host'], | |
| self._config['https-port']), | |
| WebhookRequestHandler) | |
| # Setup SSL for HTTP server | |
| self._https_server_unwrapped_socket = self._https_server.socket | |
| self._https_server.socket = ssl.wrap_socket(self._https_server.socket, | |
| keyfile=self._config['ssl-key'], | |
| certfile=self._config['ssl-cert'], | |
| server_side=True) | |
| sa = self._https_server.socket.getsockname() | |
| self._http_port = sa[1] | |
| self._server_status['https-uri'] = "https://%s:%s" % (self._config['https-host'], sa[1]) | |
| self._startup_event.log_info("Listening for connections on %s" % self._server_status['https-uri']) | |
| self._startup_event.http_address = sa[0] | |
| self._startup_event.http_port = sa[1] | |
| self._startup_event.set_http_started(True) | |
| except socket.error as e: | |
| self._startup_event.log_critical("Unable to start HTTPS server: %s" % e) | |
| return | |
| # Run forever | |
| try: | |
| self._https_server.serve_forever() | |
| except socket.error as e: | |
| event = SystemEvent() | |
| self._event_store.register_action(event) | |
| event.log_critical("Error on socket: %s" % e) | |
| sys.exit(1) | |
| except KeyboardInterrupt as e: | |
| event = SystemEvent() | |
| self._event_store.register_action(event) | |
| event.log_info('Requested close by keyboard interrupt signal') | |
| self.stop() | |
| self.exit() | |
| event = SystemEvent() | |
| self._event_store.register_action(event) | |
| event.log_info('HTTPS server did quit') | |
| def serve_wss(self): | |
| """Start a web socket server over SSL, used by the web UI to get notifications about updates.""" | |
| import os | |
| from .events import SystemEvent | |
| # Start a web socket server if the web UI is enabled | |
| if not self._config['web-ui-enabled']: | |
| return | |
| if not self._config['wss-enabled']: | |
| return | |
| if not os.path.isfile(self._config['ssl-cert']): | |
| self._startup_event.log_critical("Unable to activate SSL: File does not exist: %s" % self._config['ssl-cert']) | |
| return | |
| try: | |
| import os | |
| from autobahn.websocket import WebSocketServerProtocol, WebSocketServerFactory | |
| from twisted.internet import reactor, ssl | |
| from twisted.internet.error import BindError | |
| # Create a WebSocketClientHandler instance | |
| WebSocketClientHandler = WebSocketClientHandlerFactory(self._config, self._ws_clients, self._event_store, self._server_status) | |
| uri = u"ws://%s:%s" % (self._config['wss-host'], self._config['wss-port']) | |
| factory = WebSocketServerFactory(uri) | |
| factory.protocol = WebSocketClientHandler | |
| # factory.setProtocolOptions(maxConnections=2) | |
| # note to self: if using putChild, the child must be bytes... | |
| if self._config['ssl-key'] and self._config['ssl-cert']: | |
| contextFactory = ssl.DefaultOpenSSLContextFactory(privateKeyFileName=self._config['ssl-key'], certificateFileName=self._config['ssl-cert']) | |
| else: | |
| contextFactory = ssl.DefaultOpenSSLContextFactory(privateKeyFileName=self._config['ssl-cert'], certificateFileName=self._config['ssl-cert']) | |
| self._ws_server_port = reactor.listenSSL(self._config['wss-port'], factory, contextFactory) | |
| # self._ws_server_port = reactor.listenTCP(self._config['wss-port'], factory) | |
| self._server_status['wss-uri'] = "wss://%s:%s" % (self._config['wss-host'], self._config['wss-port']) | |
| self._startup_event.log_info("Listening for connections on %s" % self._server_status['wss-uri']) | |
| self._startup_event.ws_address = self._config['wss-host'] | |
| self._startup_event.ws_port = self._config['wss-port'] | |
| self._startup_event.set_ws_started(True) | |
| # Serve forever (until reactor.stop()) | |
| reactor.run(installSignalHandlers=False) | |
| except BindError as e: | |
| self._startup_event.log_critical("Unable to start web socket server: %s" % e) | |
| except ImportError: | |
| self._startup_event.log_error("Unable to start web socket server due to missing dependency.") | |
| event = SystemEvent() | |
| self._event_store.register_action(event) | |
| event.log_info('WSS server did quit') | |
| def serve_forever(self): | |
| """Start HTTP and web socket servers.""" | |
| import sys | |
| import socket | |
| import logging | |
| import os | |
| from .events import SystemEvent | |
| import threading | |
| try: | |
| from autobahn.websocket import WebSocketServerProtocol, WebSocketServerFactory | |
| from twisted.internet import reactor | |
| # Given that the necessary dependencies are present, notify the | |
| # event that we expect the web socket server to be started | |
| self._startup_event.ws_started = False | |
| except ImportError: | |
| pass | |
| # Notify the event that we expect the http server to be started | |
| self._startup_event.http_started = False | |
| # Add script dir to sys path, allowing us to import sub modules even after changing cwd | |
| sys.path.insert(1, os.path.dirname(os.path.realpath(__file__))) | |
| # Set CWD to public www folder. This makes the http server serve files from the wwwroot directory. | |
| wwwroot = os.path.join(os.path.dirname(os.path.realpath(__file__)), "wwwroot") | |
| os.chdir(wwwroot) | |
| threads = [ | |
| # HTTP server | |
| threading.Thread(target=self.serve_http), | |
| # HTTPS server | |
| threading.Thread(target=self.serve_https), | |
| # Web socket SSL server | |
| threading.Thread(target=self.serve_wss) | |
| ] | |
| # Start all threads | |
| for thread in threads: | |
| thread.start() | |
| # Wait for each thread to finish | |
| for thread in threads: | |
| # Wait for thread to finish without blocking main thread | |
| while thread.is_alive(): | |
| thread.join(5) | |
| def signal_handler(self, signum, frame): | |
| from .events import SystemEvent | |
| self.stop() | |
| event = SystemEvent() | |
| self._event_store.register_action(event) | |
| # Reload configuration on SIGHUP events (conventional for daemon processes) | |
| if signum == 1: | |
| self.setup(self._config) | |
| self.serve_forever() | |
| return | |
| # Keyboard interrupt signal | |
| elif signum == 2: | |
| event.log_info('Recieved keyboard interrupt signal (%s) from the OS, shutting down.' % signum) | |
| else: | |
| event.log_info('Recieved signal (%s) from the OS, shutting down.' % signum) | |
| self.exit() | |
| def stop(self): | |
| """Stop all running TCP servers (HTTP and web socket servers)""" | |
| # Stop HTTP server if running | |
| if self._http_server is not None: | |
| # Shut down the underlying TCP server | |
| self._http_server.shutdown() | |
| # Close the socket | |
| self._http_server.socket.close() | |
| # Stop HTTPS server if running | |
| if self._https_server is not None: | |
| # Shut down the underlying TCP server | |
| self._https_server.shutdown() | |
| # Close the socket | |
| self._https_server.socket.close() | |
| if self._https_server_unwrapped_socket is not None: | |
| self._https_server_unwrapped_socket.close() | |
| # Stop web socket server if running | |
| try: | |
| from twisted.internet import reactor | |
| reactor.callFromThread(reactor.stop) | |
| except ImportError: | |
| pass | |
| def exit(self): | |
| import sys | |
| import logging | |
| logger = logging.getLogger() | |
| logger.info('Goodbye') | |
| # Delete PID file | |
| self.remove_pid_file() | |
| # Restore stdin and stdout | |
| if 'intercept-stdout' in self._config and self._config['intercept-stdout']: | |
| sys.stdout = self._default_stdout | |
| sys.stderr = self._default_stderr | |
| def main(): | |
| import signal | |
| from gitautodeploy import GitAutoDeploy | |
| from cli.config import get_config_defaults, get_config_from_environment | |
| from cli.config import get_config_from_argv, find_config_file | |
| from cli.config import get_config_from_file, get_repo_config_from_environment | |
| from cli.config import init_config, get_config_file_path, rename_legacy_attribute_names | |
| from cli.config import ConfigFileNotFoundException, ConfigFileInvalidException | |
| import logging | |
| import sys | |
| import os | |
| logger = logging.getLogger() | |
| app = GitAutoDeploy() | |
| if hasattr(signal, 'SIGHUP'): | |
| signal.signal(signal.SIGHUP, app.signal_handler) | |
| if hasattr(signal, 'SIGINT'): | |
| signal.signal(signal.SIGINT, app.signal_handler) | |
| if hasattr(signal, 'SIGABRT'): | |
| signal.signal(signal.SIGABRT, app.signal_handler) | |
| if hasattr(signal, 'SIGPIPE') and hasattr(signal, 'SIG_IGN'): | |
| signal.signal(signal.SIGPIPE, signal.SIG_IGN) | |
| # Get default config values | |
| config = get_config_defaults() | |
| # Get config values from environment variables and commadn line arguments | |
| environment_config = get_config_from_environment() | |
| argv_config = get_config_from_argv(sys.argv[1:]) | |
| # Merge config values from environment variables | |
| config.update(environment_config) | |
| search_target = os.path.dirname(os.path.realpath(__file__)) | |
| config_file_path = get_config_file_path(environment_config, argv_config, search_target) | |
| # Config file path provided or found? | |
| if config_file_path: | |
| try: | |
| file_config = get_config_from_file(config_file_path) | |
| except ConfigFileNotFoundException as e: | |
| app.setup_console_logger() | |
| logger.critical("No config file not found at '%s'" % e) | |
| return | |
| except ConfigFileInvalidException as e: | |
| app.setup_console_logger() | |
| logger.critical("Unable to read config file due to invalid JSON format in '%s'" % e) | |
| return | |
| # Merge config values from config file (overrides environment variables) | |
| config.update(file_config) | |
| # Merge config value from command line (overrides environment variables and config file) | |
| config.update(argv_config) | |
| # Rename legacy config option names | |
| config = rename_legacy_attribute_names(config) | |
| # Extend config data with any repository defined by environment variables | |
| repo_config = get_repo_config_from_environment() | |
| if repo_config: | |
| if not 'repositories' in config: | |
| config['repositories'] = [] | |
| config['repositories'].append(repo_config) | |
| # Initialize config by expanding with missing values | |
| init_config(config) | |
| app.setup(config) | |
| app.serve_forever() | |