Variable substitution + refactor

README.md

@@ -13,7 +13,11 @@ The script requires, that the gitlab secret token is set! You can define the val
 The structure of the [configuration file](#example-config) requires the homepage of the gitlab project as key.
 
 ### Command
-Define, which command should be run after the hook was received.
+Define which command should be run after the hook was received.
+
+A few variables coming from the hook payload are available as variables that
+will be substituted shell-style when included in the command: `event`, `sha`,
+`user`, `ref`, `project_name` and `project_owner`.
 
 ### Example config
 ```
@@ -25,7 +29,11 @@ https://git.example.ch/exmaple/myrepo:
   gitlab_token: mysecret-myrepo
 # test-repo
 https://git.example.ch/exmaple/test-repo:
-  command: uname
+  command:
+    - /usr/local/bin/myscript
+    - $project_name
+    - $project_owner
+    - $ref
   gitlab_token: mysecret-test-repo
 ```
 
@@ -60,4 +68,4 @@ optional arguments:
   --addr ADDR  address where it listens (default: 0.0.0.0)
   --port PORT  port where it listens (default: 8666)
   --cfg CFG    path to the config file (default: config.yaml)
-```
+```

gitlab-webhook-receiver.py

@@ -4,74 +4,121 @@
 """ Gitlab Webhook Receiver """
 # Based on: https://github.com/schickling/docker-hook
 
+import sys
+import logging
 import json
-import yaml
 import subprocess
+import shlex
+from string import Template
 from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
+
 try:
     # For Python 3.0 and later
-    from http.server import HTTPServer
     from http.server import BaseHTTPRequestHandler
+    from socketserver import TCPServer
 except ImportError:
     # Fall back to Python 2
     from BaseHTTPServer import BaseHTTPRequestHandler
-    from BaseHTTPServer import HTTPServer as HTTPServer
-import sys
-import logging
+    from SocketServer import TCPServer
+
+import yaml
 
 logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
                     level=logging.DEBUG,
                     stream=sys.stdout)
 
 
-class RequestHandler(BaseHTTPRequestHandler):
-    """A POST request handler."""
+class HookHandler(BaseHTTPRequestHandler):
+    """Handle Gitlab hook POST requests and calls a command."""
 
     def do_POST(self):
-        logging.info("Hook received")
-
         # get payload
-        header_length = int(self.headers.getheader('content-length', "0"))
+        header_length = int(self.headers.get('content-length', "0"))
         json_payload = self.rfile.read(header_length)
         json_params = {}
-        if len(json_payload) > 0:
+        if json_payload:
             json_params = json.loads(json_payload)
 
-        # get gitlab secret token
-        gitlab_token_header = self.headers.getheader('X-Gitlab-Token')
-
-        # get project homepage
+        # get project configuration
         project = json_params['project']['homepage']
+        try:
+            project_config = self.server.config[project]
+        except KeyError as ker:
+            self.send_response(404, "Unknown project %s" % ker)
+            self.end_headers()
+            return
 
+        # Fetch the project's gitlab token in configuration
         try:
-            # get command and token from config file
-            command = config[project]['command']
-            gitlab_token = config[project]['gitlab_token']
+            gitlab_token = project_config['gitlab_token']
+        except KeyError as ker:
+            self.send_response(500, "No gitlab token in project configuration")
+            self.end_headers()
+            return
 
-            logging.info("Load project '%s' and run command '%s'", project, command)
+        # Check it
+        if gitlab_token != self.headers.get('X-Gitlab-Token'):
+            self.send_response(401, "Invalid token")
+            self.end_headers()
+            return
+
+        # A dict of values that can be substitued on the command line
+        available_params = {
+            "event": json_params.get("object_kind"),
+            "sha": json_params.get("checkout_sha"),
+            "user": json_params.get("user_username"),
+            "ref": json_params.get("ref"),
+            "project_name": json_params["project"]["name"],
+            "project_owner": json_params["project"]["namespace"],
+        }
+        logging.debug("Available variable substitions: %s", available_params)
+
+        # Get the command to call
+        try:
+            configured_command = project_config["command"]
+        except KeyError:
+            self.send_response(500, "No command defined for project")
+            self.end_headers()
+            return
+
+        if not isinstance(configured_command, list):
+            configured_command = shlex.split(configured_command)
+
+        try:
+            # Substitute $variable => available_params["variable"]
+            command = [Template(i).substitute(**available_params)
+                       for i in configured_command]
         except KeyError as err:
-            self.send_response(500, "KeyError")
-            logging.error("Project '%s' not found in %s", project, args.cfg)
+            self.send_response(500, "Invalid substitution %s in command" % err)
             self.end_headers()
+            self.wfile.write("Available substitutions: %s\n"
+                             % ", ".join(available_params))
             return
 
-        # Check if the gitlab token is valid
-        if gitlab_token_header == gitlab_token:
-            logging.info("Start executing '%s'" % command)
-            try:
-                # run command in background
-                subprocess.Popen(command)
-                self.send_response(200, "OK")
-            except OSError as err:
-                self.send_response(500, "OSError")
-                logging.error("Command could not run successfully.")
-                logging.error(err)
-        else:
-            logging.error("Not authorized, Gitlab_Token not authorized")
-            self.send_response(401, "Gitlab Token not authorized")
+        logging.info("Calling %s", " ".join(command))
+        try:
+            subprocess.Popen(command)
+        except OSError as ose:
+            self.send_response(500, "Could not call command: %s" % ose)
+            self.end_headers()
+            return
+        self.send_response(200, "Hook command called")
         self.end_headers()
 
 
+class WebhookServer(TCPServer, object):
+    """Like TCPServer but has a config attribute.
+
+    We subclass TCPServer directly because HTTPServer does some name resolution
+    at binding time which isn't useful and might not work.
+    """
+    allow_reuse_address = True
+
+    def __init__(self, addr, port, config):
+        super(WebhookServer, self).__init__((addr, port), HookHandler)
+        self.config = config
+
+
 def get_parser():
     """Get a command line parser."""
     parser = ArgumentParser(description=__doc__,
@@ -85,7 +132,6 @@ def get_parser():
                         dest="port",
                         type=int,
                         default=8666,
-                        metavar="PORT",
                         help="port where it listens")
     parser.add_argument("--cfg",
                         dest="cfg",
@@ -94,26 +140,22 @@ def get_parser():
     return parser
 
 
-def main(addr, port):
-    """Start a HTTPServer which waits for requests."""
-    httpd = HTTPServer((addr, port), RequestHandler)
-    httpd.serve_forever()
-
-
-if __name__ == '__main__':
-    parser = get_parser()
-
-    if len(sys.argv) == 0:
-        parser.print_help()
-        sys.exit(1)
-    args = parser.parse_args()
+def main():
+    args = get_parser().parse_args()
 
     # load config file
     try:
         with open(args.cfg, 'r') as stream:
             config = yaml.load(stream)
     except IOError as err:
-        logging.error("Config file %s could not be loaded", args.cfg)
+        logging.error("Config file %s could not be loaded: %s", args.cfg, err)
         sys.exit(1)
+    httpd = WebhookServer(args.addr, args.port, config)
+    try:
+        httpd.serve_forever()
+    except KeyboardInterrupt:
+        logging.info("caught sigint, stopping")
 
-    main(args.addr, args.port)
+
+if __name__ == '__main__':
+    main()

tests/configs/basic.yaml

@@ -0,0 +1,8 @@
+---
+# myrepo
+http://example.com/mike/diaspora:
+  command:
+    - uname
+    - -a
+  gitlab_token: test_token
+

tests/test.py

@@ -0,0 +1,126 @@
+from unittest import TestCase  # not really unit tests though
+import subprocess
+from random import randint
+from signal import SIGINT
+import requests
+from functools import partial
+from time import sleep
+import os
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+
+get_test_config = partial(os.path.join, HERE, "configs")
+
+# shamelessly copied from the gitlab docs
+sample_hook_data = """
+{
+  "object_kind": "push",
+  "before": "95790bf891e76fee5e1747ab589903a6a1f80f22",
+  "after": "da1560886d4f094c3e6c9ef40349f7d38b5d27d7",
+  "ref": "refs/heads/master",
+  "checkout_sha": "da1560886d4f094c3e6c9ef40349f7d38b5d27d7",
+  "user_id": 4,
+  "user_name": "John Smith",
+  "user_username": "jsmith",
+  "user_email": "[email protected]",
+  "user_avatar": "https://s.gravatar.com/avatar/nope",
+  "project_id": 15,
+  "project":{
+    "id": 15,
+    "name":"Diaspora",
+    "description":"",
+    "web_url":"http://example.com/mike/diaspora",
+    "avatar_url":null,
+    "git_ssh_url":"[email protected]:mike/diaspora.git",
+    "git_http_url":"http://example.com/mike/diaspora.git",
+    "namespace":"Mike",
+    "visibility_level":0,
+    "path_with_namespace":"mike/diaspora",
+    "default_branch":"master",
+    "homepage":"http://example.com/mike/diaspora",
+    "url":"[email protected]:mike/diaspora.git",
+    "ssh_url":"[email protected]:mike/diaspora.git",
+    "http_url":"http://example.com/mike/diaspora.git"
+  },
+  "repository":{
+    "name": "Diaspora",
+    "url": "[email protected]:mike/diaspora.git",
+    "description": "",
+    "homepage": "http://example.com/mike/diaspora",
+    "git_http_url":"http://example.com/mike/diaspora.git",
+    "git_ssh_url":"[email protected]:mike/diaspora.git",
+    "visibility_level":0
+  },
+  "commits": [
+    {
+      "id": "b6568db1bc1dcd7f8b4d5a946b0b91f9dacd7327",
+      "message": "Update Catalan translation to e38cb41.",
+      "timestamp": "2011-12-12T14:27:31+02:00",
+      "url": "http://example.com/mike/diaspora/commit/b6568db",
+      "author": {
+        "name": "Jordi Mallach",
+        "email": "[email protected]"
+      },
+      "added": ["CHANGELOG"],
+      "modified": ["app/controller/application.rb"],
+      "removed": []
+    },
+    {
+      "id": "da1560886d4f094c3e6c9ef40349f7d38b5d27d7",
+      "message": "fixed readme",
+      "timestamp": "2012-01-03T23:36:29+02:00",
+      "url": "http://example.com/mike/diaspora/commit/da15608",
+      "author": {
+        "name": "GitLab dev user",
+        "email": "gitlabdev@dv6700.(none)"
+      },
+      "added": ["CHANGELOG"],
+      "modified": ["app/controller/application.rb"],
+      "removed": []
+    }
+  ],
+  "total_commits_count": 4
+}"""
+
+
+class ScriptTests(TestCase):
+
+    def setUp(self):
+        # Choose a random (local)host and port
+        self.port = randint(32000, 34000)
+        self.addr = "127." + ".".join(str(randint(2, 250)) for i in range(3))
+        self.config_name = get_test_config("basic.yaml")
+        self.proc = subprocess.Popen(("python",
+                                      "./gitlab-webhook-receiver.py",
+                                      "--cfg", self.config_name,
+                                      "--addr", self.addr,
+                                      "--port", str(self.port)))
+        print("%s:%s (%s)" % (self.addr, self.port, self.config_name))
+        # wait a bit for the process to bind
+        sleep(1)
+
+    @property
+    def url(self):
+        """The running server's url"""
+        return "http://%s:%s/" % (self.addr, self.port)
+
+    def test_no_token(self):
+        """A request without a token must return 401"""
+        res = requests.post(self.url, sample_hook_data)
+        self.assertEqual(res.status_code, 401)
+
+    def test_wrong_token(self):
+        """A request with the wrong token must return 401"""
+        res = requests.post(self.url, sample_hook_data,
+                            headers={"X-Gitlab-Token": "lol"})
+        self.assertEqual(res.status_code, 401)
+
+    def test_right_token(self):
+        """A request with the wrong token must return 200"""
+        res = requests.post(self.url, sample_hook_data,
+                            headers={"X-Gitlab-Token": "test_token"})
+        self.assertEqual(res.status_code, 200, res.reason)
+
+    def tearDown(self):
+        self.proc.send_signal(SIGINT)
+        self.proc.communicate()

tox.ini

@@ -0,0 +1,11 @@
+[tox]
+envlist = py27,py36
+skipsdist = true
+
+[testenv]
+deps = pyyaml
+       requests
+       flake8
+
+commands = python -m unittest discover tests
+           flake8