Made naming rules more lax and migration script more stable. (#70)

Author: kodek16

filetracker/scripts/migrate.py

@@ -6,16 +6,19 @@
 
 import argparse
 import os
+import sys
 
-from filetracker.client import Client
+from filetracker.client import Client, FiletrackerError
 from filetracker.scripts import progress_bar
 
 # Value used for aligning printed action names
 _ACTION_LENGTH = 25
 
 
 _DESCRIPTION = """
-Uploads all files to a remote filetracker server.
+Uploads files to a remote filetracker server.
+
+Use --root parameter to upload only parts of the storage.
 
 The intention for this script is to support migration to new filetracker
 servers that change the format of disk storage.
@@ -25,15 +28,24 @@
 """
 
 
-def main():
+def main(args=None):
     parser = argparse.ArgumentParser(description=_DESCRIPTION)
-    parser.add_argument('files', help='root of the file tree to be uploaded')
+    parser.add_argument('files', help='file tree to be uploaded')
     parser.add_argument('url', help='URL of the filetracker server')
+    parser.add_argument('--root',
+            help='the directory that corresponds to the storage root')
     parser.add_argument('-s', '--silent', action='store_true',
             help='if set, progress bar is not printed')
 
-    args = parser.parse_args()
-    root, url, silent = args.files, args.url, args.silent
+    args = parser.parse_args(args)
+
+    upload_root = args.files
+    url = args.url
+    storage_root = args.root
+    silent = args.silent
+
+    if storage_root is None:
+        storage_root = upload_root
 
     # Create a client without local cache.
     client = Client(local_store=None, remote_url=url)
@@ -50,7 +62,7 @@ def main():
 
     with progress_bar.conditional(show=not silent,
                                   widgets=size_widgets) as bar:
-        for cur_dir, _, files in os.walk(root):
+        for cur_dir, _, files in os.walk(upload_root):
             for file_name in files:
                 total_size += os.path.getsize(os.path.join(cur_dir, file_name))
                 bar.update(total_size)
@@ -69,18 +81,22 @@ def main():
     with progress_bar.conditional(show=not silent,
                                   max_value=total_size,
                                   widgets=upload_widgets) as bar:
-        for cur_dir, _, files in os.walk(root):
+        for cur_dir, _, files in os.walk(upload_root):
             for file_name in files:
                 file_path = os.path.join(cur_dir, file_name)
-                remote_path = '/' + os.path.relpath(file_path, root)
+                remote_path = '/' + os.path.relpath(file_path, storage_root)
 
                 file_stat = os.stat(file_path)
                 file_size = file_stat.st_size
                 file_version = int(file_stat.st_mtime)
 
                 remote_name = '{}@{}'.format(remote_path, file_version)
 
-                client.put_file(remote_name, file_path, to_local_store=False)
+                try:
+                    client.put_file(remote_name, file_path, to_local_store=False)
+                except FiletrackerError as e:
+                    print('ERROR when uploading {}:\n{}'.format(file_path, e),
+                          file=sys.stderr)
 
                 processed_size += file_size
                 bar.update(processed_size)

filetracker/scripts/migrate_test.py

@@ -0,0 +1,69 @@
+"""Tests for migrate script."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from multiprocessing import Process
+import os
+import shutil
+import tempfile
+import time
+import unittest
+
+from filetracker.client import Client, FiletrackerError
+from filetracker.scripts import migrate
+from filetracker.servers.run import main as server_main
+
+_TEST_PORT_NUMBER = 45785
+
+
+class MigrateScriptTest(unittest.TestCase):
+    def setUp(self):
+        self.temp_dir = tempfile.mkdtemp()
+        os.makedirs(os.path.join(self.temp_dir, 'old_root', 'foo', 'bar'))
+        os.makedirs(os.path.join(self.temp_dir, 'new_root'))
+
+        self.server_process = Process(
+                target=_start_server,
+                args=(os.path.join(self.temp_dir, 'new_root'),))
+        self.server_process.start()
+        time.sleep(2)
+
+        self.server_url = 'http://127.0.0.1:{}'.format(_TEST_PORT_NUMBER)
+        self.client = Client(local_store=None, remote_url=self.server_url)
+
+    def tearDown(self):
+        self.server_process.terminate()
+        shutil.rmtree(self.temp_dir)
+
+    def test_should_upload_files_with_correct_relative_root(self):
+        _touch(os.path.join(self.temp_dir, 'old_root', 'foo', 'a.txt'))
+        _touch(os.path.join(self.temp_dir, 'old_root', 'foo', 'bar', 'b.txt'))
+        _touch(os.path.join(self.temp_dir, 'old_root', 'c.txt'))
+        _touch(os.path.join(self.temp_dir, 'old_root', 'd.txt'))
+
+        migrate.main([
+            os.path.join(self.temp_dir, 'old_root', 'foo'),
+            self.server_url,
+            '--root',
+            os.path.join(self.temp_dir, 'old_root'),
+            '-s'])
+
+        self.assertEqual(self.client.get_stream('/foo/a.txt')[0].read(), b'')
+        self.assertEqual(self.client.get_stream('/foo/bar/b.txt')[0].read(), b'')
+
+        with self.assertRaises(FiletrackerError):
+            self.client.get_stream('/c.txt')
+
+        with self.assertRaises(FiletrackerError):
+            self.client.get_stream('/d.txt')
+
+
+def _start_server(server_dir):
+    server_main(['-p', str(_TEST_PORT_NUMBER), '-d', server_dir, '-D',
+                 '--workers', '4'])
+
+def _touch(path):
+    with open(path, 'w') as f:
+        pass

filetracker/servers/base.py

@@ -67,11 +67,11 @@ def get_endpoint_and_path(environ):
     of them are without leading slashes.
     """
     path = environ['PATH_INFO']
-    if '..' in path:
-        raise HttpError('400 Bad Request', 'Path cannot contain "..".')
-
     components = path.split('/')
 
+    if '..' in components:
+        raise HttpError('400 Bad Request', 'Path cannot contain "..".')
+
     # Strip closing slash
     if components and components[-1] == '':
         components.pop()

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 setup(
     name = 'filetracker',
-    version = '2.1.1',
+    version = '2.1.2',
     author = 'SIO2 Project Team',
     author_email = '[email protected]',
     description = 'Filetracker caching file storage',