Docker rebase

Allows to rebase docker image on another parent. v2 support only (no means to test on v1).

Author: arcivanov

.travis.yml

@@ -2,7 +2,10 @@ language: python
 python:
   - "2.7"
   - "3.4"
+  - "3.5"
+  - "3.6"
   - "pypy"
+  - "pypy3"
 install: "pip install -r requirements.txt"
 script:
   - "py.test -v"

Makefile

@@ -12,6 +12,12 @@ test-py34: prepare
 test-py35: prepare
 	tox -e py35 -- tests
 
+test-py36: prepare
+	tox -e py36 -- tests
+
+test-py37: prepare
+	tox -e py37 -- tests
+
 test-unit: prepare
 	tox -- tests/test_unit*
 
@@ -28,6 +34,11 @@ else
 	@sudo chmod +x /usr/bin/docker
 endif
 
+ci-install-pythons:
+	curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
+	pyenv update
+	for pyver in 2.7.14 3.4.8 3.5.5 3.6.5; do pyenv install -s $$pyver; done
+
 ci-publish-junit:
 	@mkdir -p ${CIRCLE_TEST_REPORTS}
 	@cp target/junit*.xml ${CIRCLE_TEST_REPORTS}

circle.yml

@@ -12,9 +12,14 @@ machine:
     CI: true
 
 dependencies:
+  pre:
+    - mkdir -p ~/.pyenv
+    - make -f Makefile ci-install-pythons
+  cache_directories:
+    - ~/.pyenv
   override:
     - pip install tox tox-pyenv docker-py>=1.7.2 six
-    - pyenv local 2.7.11 3.4.4 3.5.1
+    - pyenv local 2.7.14 3.4.8 3.5.5 3.6.5
   post:
     - docker version
     - docker info
@@ -27,5 +32,7 @@ test:
         parallel: true
     - make test-py35:
         parallel: true
+    - make test-py36:
+        parallel: true
   post:
     - make ci-publish-junit

docker_squash/cli.py

@@ -60,18 +60,22 @@ def run(self):
             '--version', action='version', help='Show version and exit', version=version)
 
         parser.add_argument('image', help='Image to be squashed')
-        parser.add_argument(
-            '-d', '--development', action='store_true', help='Does not clean up after failure for easier debugging')
-        parser.add_argument(
-            '-f', '--from-layer', help='ID of the layer or image ID or image name. If not specified will squash all layers in the image')
-        parser.add_argument(
-            '-t', '--tag', help="Specify the tag to be used for the new image. If not specified no tag will be applied")
-        parser.add_argument(
-            '-c', '--cleanup', action='store_true', help="Remove source image from Docker after squashing")
-        parser.add_argument(
-            '--tmp-dir', help='Temporary directory to be created and used')
-        parser.add_argument(
-            '--output-path', help='Path where the image should be stored after squashing. If not provided, image will be loaded into Docker daemon')
+        parser.add_argument('-r', '--rebase',
+                            help='Rebase the image on a different "FROM"')
+        parser.add_argument('-d', '--development', action='store_true',
+                            help='Does not clean up after failure for easier debugging')
+        parser.add_argument('-f', '--from-layer',
+                            help='ID of the layer or image ID or image name. '
+                                 'If not specified will squash all layers in the image')
+        parser.add_argument('-t', '--tag',
+                            help="Specify the tag to be used for the new image. If not specified no tag will be applied")
+        parser.add_argument('-c', '--cleanup', action='store_true',
+                            help="Remove source image from Docker after squashing")
+        parser.add_argument('--tmp-dir',
+                            help='Temporary directory to be created and used')
+        parser.add_argument('--output-path',
+                            help='Path where the image should be stored after squashing. '
+                                 'If not provided, image will be loaded into Docker daemon')
 
         args = parser.parse_args()
 
@@ -84,7 +88,8 @@ def run(self):
 
         try:
             squash.Squash(log=self.log, image=args.image,
-                          from_layer=args.from_layer, tag=args.tag, output_path=args.output_path, tmp_dir=args.tmp_dir, development=args.development, cleanup=args.cleanup).run()
+                          from_layer=args.from_layer, tag=args.tag, output_path=args.output_path, tmp_dir=args.tmp_dir,
+                          development=args.development, cleanup=args.cleanup, rebase=args.rebase).run()
         except KeyboardInterrupt:
             self.log.error("Program interrupted by user, exiting...")
             sys.exit(1)
@@ -96,8 +101,9 @@ def run(self):
             else:
                 self.log.error(str(e))
 
-            self.log.error(
-                "Execution failed, consult logs above. If you think this is our fault, please file an issue: https://github.com/goldmann/docker-squash/issues, thanks!")
+            self.log.error("Execution failed, consult logs above. "
+                           "If you think this is our fault, please file an issue: "
+                           "https://github.com/goldmann/docker-squash/issues, thanks!")
 
             if isinstance(e, SquashError):
                 sys.exit(e.code)

docker_squash/image.py

@@ -42,17 +42,19 @@ class Image(object):
     FORMAT = None
     """ Image format version """
 
-    def __init__(self, log, docker, image, from_layer, tmp_dir=None, tag=None):
+    def __init__(self, log, docker, image, from_layer, tmp_dir=None, tag=None, rebase=None):
         self.log = log
         self.debug = self.log.isEnabledFor(logging.DEBUG)
         self.docker = docker
         self.image = image
         self.from_layer = from_layer
         self.tag = tag
+        self.rebase = rebase
         self.image_name = None
         self.image_tag = None
         self.squash_id = None
 
+
         # Workaround for https://play.golang.org/p/sCsWMXYxqy
         #
         # Golang doesn't add padding to microseconds when marshaling
@@ -69,6 +71,7 @@ def __init__(self, log, docker, image, from_layer, tmp_dir=None, tag=None):
 
     def squash(self):
         self._before_squashing()
+        self.log.info("Squashing image '%s'..." % self.image)
         ret = self._squash()
         self._after_squashing()
 
@@ -92,12 +95,14 @@ def _initialize_directories(self):
 
         # Temporary location on the disk of the old, unpacked *image*
         self.old_image_dir = os.path.join(self.tmp_dir, "old")
+        # Temporary location on the disk of the rebase, unpacked *image*
+        self.rebase_image_dir = os.path.join(self.tmp_dir, "rebase")
         # Temporary location on the disk of the new, unpacked, squashed *image*
         self.new_image_dir = os.path.join(self.tmp_dir, "new")
         # Temporary location on the disk of the squashed *layer*
         self.squashed_dir = os.path.join(self.new_image_dir, "squashed")
 
-        for d in self.old_image_dir, self.new_image_dir:
+        for d in self.old_image_dir, self.new_image_dir, self.rebase_image_dir:
             os.makedirs(d)
 
     def _squash_id(self, layer):
@@ -150,8 +155,16 @@ def _before_squashing(self):
         try:
             self.old_image_id = self.docker.inspect_image(self.image)['Id']
         except SquashError:
-            raise SquashError(
-                "Could not get the image ID to squash, please check provided 'image' argument: %s" % self.image)
+            raise SquashError("Could not get the image ID to squash, "
+                              "please check provided 'image' argument: %s" % self.image)
+
+        if self.rebase:
+            # The image id or name of the image to rebase to
+            try:
+                self.rebase = self.docker.inspect_image(self.rebase)['Id']
+            except SquashError:
+                raise SquashError("Could not get the image ID to rebase to, "
+                                  "please check provided 'rebase' argument: %s" % self.rebase)
 
         self.old_image_layers = []
 
@@ -164,32 +177,36 @@ def _before_squashing(self):
         self.log.debug("Old layers: %s", self.old_image_layers)
 
         # By default - squash all layers.
-        if self.from_layer == None:
+        if self.from_layer is None:
             self.from_layer = len(self.old_image_layers)
 
         try:
             number_of_layers = int(self.from_layer)
 
-            self.log.debug(
-                "We detected number of layers as the argument to squash")
+            self.log.debug("We detected number of layers as the argument to squash")
         except ValueError:
             self.log.debug("We detected layer as the argument to squash")
 
             squash_id = self._squash_id(self.from_layer)
 
             if not squash_id:
-                raise SquashError(
-                    "The %s layer could not be found in the %s image" % (self.from_layer, self.image))
+                raise SquashError("The %s layer could not be found in the %s image" % (self.from_layer, self.image))
 
-            number_of_layers = len(self.old_image_layers) - \
-                self.old_image_layers.index(squash_id) - 1
+            number_of_layers = len(self.old_image_layers) - self.old_image_layers.index(squash_id) - 1
 
         self._validate_number_of_layers(number_of_layers)
 
         marker = len(self.old_image_layers) - number_of_layers
 
         self.layers_to_squash = self.old_image_layers[marker:]
-        self.layers_to_move = self.old_image_layers[:marker]
+        if self.rebase:
+            self.layers_to_move = []
+            self._read_layers(self.layers_to_move, self.rebase)
+            self.layers_to_move.reverse()
+        else:
+            self.layers_to_move = self.old_image_layers[:marker]
+
+        self.old_image_squash_marker = marker
 
         self.log.info("Checking if squashing is necessary...")
 
@@ -199,18 +216,18 @@ def _before_squashing(self):
         if len(self.layers_to_squash) == 1:
             raise SquashUnnecessaryError("Single layer marked to squash, no squashing is required")
 
-        self.log.info("Attempting to squash last %s layers...",
-                      number_of_layers)
+        self.log.info("Attempting to squash last %s layers%s...", number_of_layers,
+                      " rebasing on %s" % self.rebase if self.rebase else "")
         self.log.debug("Layers to squash: %s", self.layers_to_squash)
         self.log.debug("Layers to move: %s", self.layers_to_move)
 
         # Fetch the image and unpack it on the fly to the old image directory
         self._save_image(self.old_image_id, self.old_image_dir)
+        if self.rebase:
+            self._save_image(self.rebase, self.rebase_image_dir)
 
         self.size_before = self._dir_size(self.old_image_dir)
 
-        self.log.info("Squashing image '%s'..." % self.image)
-
     def _after_squashing(self):
         self.log.debug("Removing from disk already squashed layers...")
         shutil.rmtree(self.old_image_dir, ignore_errors=True)
@@ -670,7 +687,7 @@ def _squash_layers(self, layers_to_squash, layers_to_move):
 
         # Find all files in layers that we don't squash
         files_in_layers_to_move = self._files_in_layers(
-            layers_to_move, self.old_image_dir)
+            layers_to_move, self.old_image_dir if not self.rebase else self.rebase_image_dir)
 
         with tarfile.open(self.squashed_tar, 'w', format=tarfile.PAX_FORMAT) as squashed_tar:
             to_skip = []

docker_squash/squash.py

@@ -14,7 +14,7 @@
 class Squash(object):
 
     def __init__(self, log, image, docker=None, from_layer=None, tag=None, tmp_dir=None,
-                 output_path=None, load_image=True, development=False, cleanup=False):
+                 output_path=None, load_image=True, development=False, cleanup=False, rebase=None):
         self.log = log
         self.docker = docker
         self.image = image
@@ -25,6 +25,7 @@ def __init__(self, log, image, docker=None, from_layer=None, tag=None, tmp_dir=N
         self.load_image = load_image
         self.development = development
         self.cleanup = cleanup
+        self.rebase = rebase
 
         if not docker:
             self.docker = common.docker_client(self.log)
@@ -48,10 +49,10 @@ def run(self):
 
         if StrictVersion(docker_version['ApiVersion']) >= StrictVersion("1.22"):
             image = V2Image(self.log, self.docker, self.image,
-                            self.from_layer, self.tmp_dir, self.tag)
+                            self.from_layer, self.tmp_dir, self.tag, self.rebase)
         else:
             image = V1Image(self.log, self.docker, self.image,
-                            self.from_layer, self.tmp_dir, self.tag)
+                            self.from_layer, self.tmp_dir, self.tag, self.rebase)
 
         self.log.info("Using %s image format" % image.FORMAT)
 

docker_squash/v1_image.py

@@ -33,7 +33,7 @@ def _squash(self):
         self._write_version_file(self.squashed_dir)
         # Move all the layers that should be untouched
         self._move_layers(self.layers_to_move,
-                          self.old_image_dir, self.new_image_dir)
+                          self.old_image_dir if not self.rebase else self.rebase_image_dir, self.new_image_dir)
 
         config_file = os.path.join(
             self.old_image_dir, self.old_image_id, "json")