From 2af609a61610ea061d05ca74716f782ec67c0d29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Noss?= Date: Fri, 29 Jun 2018 15:38:55 +0200 Subject: [PATCH 1/2] Add variable substitution support in commands, start refactoring --- gitlab-webhook-receiver.py | 111 +++++++++++++++++++++++-------------- 1 file changed, 69 insertions(+), 42 deletions(-) diff --git a/gitlab-webhook-receiver.py b/gitlab-webhook-receiver.py index cdde750..be02697 100755 --- a/gitlab-webhook-receiver.py +++ b/gitlab-webhook-receiver.py @@ -4,10 +4,14 @@ """ Gitlab Webhook Receiver """ # Based on: https://github.com/schickling/docker-hook +import sys +import logging import json -import yaml import subprocess + +from string import Template from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter + try: # For Python 3.0 and later from http.server import HTTPServer @@ -16,8 +20,10 @@ # Fall back to Python 2 from BaseHTTPServer import BaseHTTPRequestHandler from BaseHTTPServer import HTTPServer as HTTPServer -import sys -import logging + +import yaml + +CONFIG = None logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.DEBUG, @@ -34,41 +40,67 @@ def do_POST(self): header_length = int(self.headers.getheader('content-length', "0")) json_payload = self.rfile.read(header_length) json_params = {} - if len(json_payload) > 0: + if json_payload: json_params = json.loads(json_payload) - # get gitlab secret token - gitlab_token_header = self.headers.getheader('X-Gitlab-Token') - - # get project homepage + # get project configuration project = json_params['project']['homepage'] + try: + project_config = CONFIG[project] + except KeyError as ker: + self.send_response(400, "Unknown project %r" % ker) + self.end_headers() + return + # Fetch the project's gitlab token in configuration try: - # get command and token from config file - command = config[project]['command'] - gitlab_token = config[project]['gitlab_token'] + gitlab_token = project_config['gitlab_token'] + except KeyError as ker: + self.send_response(500, "No gitlab token in project configuration") + self.end_headers() + return - logging.info("Load project '%s' and run command '%s'", project, command) + # Check it + if gitlab_token != self.headers.getheader('X-Gitlab-Token'): + self.send_response(401, "Invalid token") + self.end_headers() + return + + # A dict of values that can be substitued on the command line + available_params = { + "event": json_params.get("object_kind"), + "sha": json_params.get("checkout_sha"), + "user": json_params.get("user_username"), + "ref": json_params.get("ref"), + "project_name": json_params["project"]["name"], + "project_owner": json_params["project"]["namespace"], + } + logging.debug("Available variable substitions: %s", available_params) + + # Get the command to call + try: + # Substitute $variable => available_params["variable"] + command = [Template(i).substitute(**available_params) + for i in project_config.get("command", ())] except KeyError as err: - self.send_response(500, "KeyError") - logging.error("Project '%s' not found in %s", project, args.cfg) + self.send_response(500, "Invalid substitution %s in command" % err) + self.end_headers() + self.wfile.write("Available substitutions: %s\n" + % ", ".join(available_params)) + return + if not command: + self.send_response(500, "No command defined for project") self.end_headers() return - # Check if the gitlab token is valid - if gitlab_token_header == gitlab_token: - logging.info("Start executing '%s'" % command) - try: - # run command in background - subprocess.Popen(command) - self.send_response(200, "OK") - except OSError as err: - self.send_response(500, "OSError") - logging.error("Command could not run successfully.") - logging.error(err) - else: - logging.error("Not authorized, Gitlab_Token not authorized") - self.send_response(401, "Gitlab Token not authorized") + logging.info("Calling %s", " ".join(command)) + try: + subprocess.Popen(command) + except OSError as ose: + self.send_response(500, "Could not call command: %s" % ose) + self.end_headers() + return + self.send_response(200, "Hook command called") self.end_headers() @@ -85,7 +117,6 @@ def get_parser(): dest="port", type=int, default=8666, - metavar="PORT", help="port where it listens") parser.add_argument("--cfg", dest="cfg", @@ -94,26 +125,22 @@ def get_parser(): return parser -def main(addr, port): - """Start a HTTPServer which waits for requests.""" - httpd = HTTPServer((addr, port), RequestHandler) - httpd.serve_forever() - - -if __name__ == '__main__': +def main(): + global CONFIG # FIXME XXX TODO YUCK parser = get_parser() - if len(sys.argv) == 0: - parser.print_help() - sys.exit(1) args = parser.parse_args() # load config file try: with open(args.cfg, 'r') as stream: - config = yaml.load(stream) + CONFIG = yaml.load(stream) except IOError as err: - logging.error("Config file %s could not be loaded", args.cfg) + logging.error("Config file %s could not be loaded: %s", args.cfg, err) sys.exit(1) + httpd = HTTPServer((args.addr, args.port), RequestHandler) + httpd.serve_forever() - main(args.addr, args.port) + +if __name__ == '__main__': + main() From a8c1c9d30271038c848e529f18cb5934e4211099 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Noss?= Date: Fri, 29 Jun 2018 23:47:57 +0200 Subject: [PATCH 2/2] refactor, add tests, mention variable substitution in readme --- README.md | 14 ++++- gitlab-webhook-receiver.py | 65 +++++++++++-------- tests/configs/basic.yaml | 8 +++ tests/test.py | 126 +++++++++++++++++++++++++++++++++++++ tox.ini | 11 ++++ 5 files changed, 196 insertions(+), 28 deletions(-) create mode 100644 tests/configs/basic.yaml create mode 100644 tests/test.py create mode 100644 tox.ini diff --git a/README.md b/README.md index fdf35c2..6f8529e 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,11 @@ The script requires, that the gitlab secret token is set! You can define the val The structure of the [configuration file](#example-config) requires the homepage of the gitlab project as key. ### Command -Define, which command should be run after the hook was received. +Define which command should be run after the hook was received. + +A few variables coming from the hook payload are available as variables that +will be substituted shell-style when included in the command: `event`, `sha`, +`user`, `ref`, `project_name` and `project_owner`. ### Example config ``` @@ -25,7 +29,11 @@ https://git.example.ch/exmaple/myrepo: gitlab_token: mysecret-myrepo # test-repo https://git.example.ch/exmaple/test-repo: - command: uname + command: + - /usr/local/bin/myscript + - $project_name + - $project_owner + - $ref gitlab_token: mysecret-test-repo ``` @@ -60,4 +68,4 @@ optional arguments: --addr ADDR address where it listens (default: 0.0.0.0) --port PORT port where it listens (default: 8666) --cfg CFG path to the config file (default: config.yaml) -``` \ No newline at end of file +``` diff --git a/gitlab-webhook-receiver.py b/gitlab-webhook-receiver.py index be02697..4e82cf4 100755 --- a/gitlab-webhook-receiver.py +++ b/gitlab-webhook-receiver.py @@ -8,36 +8,32 @@ import logging import json import subprocess - +import shlex from string import Template from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter try: # For Python 3.0 and later - from http.server import HTTPServer from http.server import BaseHTTPRequestHandler + from socketserver import TCPServer except ImportError: # Fall back to Python 2 from BaseHTTPServer import BaseHTTPRequestHandler - from BaseHTTPServer import HTTPServer as HTTPServer + from SocketServer import TCPServer import yaml -CONFIG = None - logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.DEBUG, stream=sys.stdout) -class RequestHandler(BaseHTTPRequestHandler): - """A POST request handler.""" +class HookHandler(BaseHTTPRequestHandler): + """Handle Gitlab hook POST requests and calls a command.""" def do_POST(self): - logging.info("Hook received") - # get payload - header_length = int(self.headers.getheader('content-length', "0")) + header_length = int(self.headers.get('content-length', "0")) json_payload = self.rfile.read(header_length) json_params = {} if json_payload: @@ -46,9 +42,9 @@ def do_POST(self): # get project configuration project = json_params['project']['homepage'] try: - project_config = CONFIG[project] + project_config = self.server.config[project] except KeyError as ker: - self.send_response(400, "Unknown project %r" % ker) + self.send_response(404, "Unknown project %s" % ker) self.end_headers() return @@ -61,7 +57,7 @@ def do_POST(self): return # Check it - if gitlab_token != self.headers.getheader('X-Gitlab-Token'): + if gitlab_token != self.headers.get('X-Gitlab-Token'): self.send_response(401, "Invalid token") self.end_headers() return @@ -78,20 +74,26 @@ def do_POST(self): logging.debug("Available variable substitions: %s", available_params) # Get the command to call + try: + configured_command = project_config["command"] + except KeyError: + self.send_response(500, "No command defined for project") + self.end_headers() + return + + if not isinstance(configured_command, list): + configured_command = shlex.split(configured_command) + try: # Substitute $variable => available_params["variable"] command = [Template(i).substitute(**available_params) - for i in project_config.get("command", ())] + for i in configured_command] except KeyError as err: self.send_response(500, "Invalid substitution %s in command" % err) self.end_headers() self.wfile.write("Available substitutions: %s\n" % ", ".join(available_params)) return - if not command: - self.send_response(500, "No command defined for project") - self.end_headers() - return logging.info("Calling %s", " ".join(command)) try: @@ -104,6 +106,19 @@ def do_POST(self): self.end_headers() +class WebhookServer(TCPServer, object): + """Like TCPServer but has a config attribute. + + We subclass TCPServer directly because HTTPServer does some name resolution + at binding time which isn't useful and might not work. + """ + allow_reuse_address = True + + def __init__(self, addr, port, config): + super(WebhookServer, self).__init__((addr, port), HookHandler) + self.config = config + + def get_parser(): """Get a command line parser.""" parser = ArgumentParser(description=__doc__, @@ -126,20 +141,20 @@ def get_parser(): def main(): - global CONFIG # FIXME XXX TODO YUCK - parser = get_parser() - - args = parser.parse_args() + args = get_parser().parse_args() # load config file try: with open(args.cfg, 'r') as stream: - CONFIG = yaml.load(stream) + config = yaml.load(stream) except IOError as err: logging.error("Config file %s could not be loaded: %s", args.cfg, err) sys.exit(1) - httpd = HTTPServer((args.addr, args.port), RequestHandler) - httpd.serve_forever() + httpd = WebhookServer(args.addr, args.port, config) + try: + httpd.serve_forever() + except KeyboardInterrupt: + logging.info("caught sigint, stopping") if __name__ == '__main__': diff --git a/tests/configs/basic.yaml b/tests/configs/basic.yaml new file mode 100644 index 0000000..e89e953 --- /dev/null +++ b/tests/configs/basic.yaml @@ -0,0 +1,8 @@ +--- +# myrepo +http://example.com/mike/diaspora: + command: + - uname + - -a + gitlab_token: test_token + diff --git a/tests/test.py b/tests/test.py new file mode 100644 index 0000000..01e156d --- /dev/null +++ b/tests/test.py @@ -0,0 +1,126 @@ +from unittest import TestCase # not really unit tests though +import subprocess +from random import randint +from signal import SIGINT +import requests +from functools import partial +from time import sleep +import os + +HERE = os.path.dirname(os.path.abspath(__file__)) + +get_test_config = partial(os.path.join, HERE, "configs") + +# shamelessly copied from the gitlab docs +sample_hook_data = """ +{ + "object_kind": "push", + "before": "95790bf891e76fee5e1747ab589903a6a1f80f22", + "after": "da1560886d4f094c3e6c9ef40349f7d38b5d27d7", + "ref": "refs/heads/master", + "checkout_sha": "da1560886d4f094c3e6c9ef40349f7d38b5d27d7", + "user_id": 4, + "user_name": "John Smith", + "user_username": "jsmith", + "user_email": "john@example.com", + "user_avatar": "https://s.gravatar.com/avatar/nope", + "project_id": 15, + "project":{ + "id": 15, + "name":"Diaspora", + "description":"", + "web_url":"http://example.com/mike/diaspora", + "avatar_url":null, + "git_ssh_url":"git@example.com:mike/diaspora.git", + "git_http_url":"http://example.com/mike/diaspora.git", + "namespace":"Mike", + "visibility_level":0, + "path_with_namespace":"mike/diaspora", + "default_branch":"master", + "homepage":"http://example.com/mike/diaspora", + "url":"git@example.com:mike/diaspora.git", + "ssh_url":"git@example.com:mike/diaspora.git", + "http_url":"http://example.com/mike/diaspora.git" + }, + "repository":{ + "name": "Diaspora", + "url": "git@example.com:mike/diaspora.git", + "description": "", + "homepage": "http://example.com/mike/diaspora", + "git_http_url":"http://example.com/mike/diaspora.git", + "git_ssh_url":"git@example.com:mike/diaspora.git", + "visibility_level":0 + }, + "commits": [ + { + "id": "b6568db1bc1dcd7f8b4d5a946b0b91f9dacd7327", + "message": "Update Catalan translation to e38cb41.", + "timestamp": "2011-12-12T14:27:31+02:00", + "url": "http://example.com/mike/diaspora/commit/b6568db", + "author": { + "name": "Jordi Mallach", + "email": "jordi@softcatala.org" + }, + "added": ["CHANGELOG"], + "modified": ["app/controller/application.rb"], + "removed": [] + }, + { + "id": "da1560886d4f094c3e6c9ef40349f7d38b5d27d7", + "message": "fixed readme", + "timestamp": "2012-01-03T23:36:29+02:00", + "url": "http://example.com/mike/diaspora/commit/da15608", + "author": { + "name": "GitLab dev user", + "email": "gitlabdev@dv6700.(none)" + }, + "added": ["CHANGELOG"], + "modified": ["app/controller/application.rb"], + "removed": [] + } + ], + "total_commits_count": 4 +}""" + + +class ScriptTests(TestCase): + + def setUp(self): + # Choose a random (local)host and port + self.port = randint(32000, 34000) + self.addr = "127." + ".".join(str(randint(2, 250)) for i in range(3)) + self.config_name = get_test_config("basic.yaml") + self.proc = subprocess.Popen(("python", + "./gitlab-webhook-receiver.py", + "--cfg", self.config_name, + "--addr", self.addr, + "--port", str(self.port))) + print("%s:%s (%s)" % (self.addr, self.port, self.config_name)) + # wait a bit for the process to bind + sleep(1) + + @property + def url(self): + """The running server's url""" + return "http://%s:%s/" % (self.addr, self.port) + + def test_no_token(self): + """A request without a token must return 401""" + res = requests.post(self.url, sample_hook_data) + self.assertEqual(res.status_code, 401) + + def test_wrong_token(self): + """A request with the wrong token must return 401""" + res = requests.post(self.url, sample_hook_data, + headers={"X-Gitlab-Token": "lol"}) + self.assertEqual(res.status_code, 401) + + def test_right_token(self): + """A request with the wrong token must return 200""" + res = requests.post(self.url, sample_hook_data, + headers={"X-Gitlab-Token": "test_token"}) + self.assertEqual(res.status_code, 200, res.reason) + + def tearDown(self): + self.proc.send_signal(SIGINT) + self.proc.communicate() diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..5a1eff6 --- /dev/null +++ b/tox.ini @@ -0,0 +1,11 @@ +[tox] +envlist = py27,py36 +skipsdist = true + +[testenv] +deps = pyyaml + requests + flake8 + +commands = python -m unittest discover tests + flake8