Update archive script

Author: szarnyasg

scripts/archive_docs.py

@@ -1,176 +1,119 @@
 import sys
 import os
 import shutil
-import datetime
-import subprocess
 import frontmatter
 
+if len(sys.argv) < 2:
+    print("Usage: archive_docs VERSION")
+    exit(1)
 
-# Usage instructions:
-# Update _config.yml to specify the new version number
-# Add a new row to /_data/versions.csv with the new version number
-# Run this script. More options below, but as an example:
-# 	run this script in the top level docs directory like: python3 scripts/archive_docs.py 1.0
-# When testing, run a clean (non-incremental) serve: jekyll serve
+old_stable_version = sys.argv[1]
 
-if len(sys.argv) < 2:
-    print(
-        "Usage: python3 scripts/archive_docs.py [version] [--noconfirm] [--date=YYYY-MM-DD]"
+
+# update new stable file based on old stable file
+def update_new_stable_page(new_stable_file, old_stable_dir):
+    # if the old counterpart exists, parse YAML metadata and get the "redirect_from" field
+    old_stable_file = new_stable_file.replace("docs/preview", "docs/stable")
+    if os.path.exists(old_stable_file):
+        old_stable_doc = frontmatter.load(old_stable_file)
+        redirect_from_field = old_stable_doc.get("redirect_from")
+    else:
+        redirect_from_field = None
+
+    new_stable_doc = frontmatter.load(new_stable_file)
+    
+    # overwrite the new stable doc's redirect_from field with the one from the old stable document
+    new_stable_doc["redirect_from"] = redirect_from_field
+
+    # replace link tags in the content
+    new_stable_doc.content = new_stable_doc.content.replace(
+        f"{{% link docs/preview/",
+        f"{{% link docs/stable/"
     )
-    print(
-        "If date is specified, this script will copy docs that existed at that specific date"
+    return frontmatter.dumps(new_stable_doc)
+
+
+# copy docs/preview to docs/stable, while keeping the redirects from docs/stable
+def archive_preview(old_stable_version):
+    src = "docs/preview"
+    dst = f"docs/stable_temp"
+    old_stable = "docs/stable"
+
+    os.makedirs(dst, exist_ok=True)
+
+    for root, dirs, files in os.walk(src):
+        rel_path = os.path.relpath(root, src)
+        dest_dir = os.path.join(dst, rel_path)
+        os.makedirs(dest_dir, exist_ok=True)
+
+        for file in files:
+            src_file = os.path.join(root, file)
+            dst_file = os.path.join(dest_dir, file)
+
+            if src_file.endswith(".md"):
+                new_content = update_new_stable_page(src_file, old_stable)
+                print(new_content)
+                with open(dst_file, 'w') as f:
+                    f.write(new_content)
+            else:
+                shutil.copy2(src_file, dst_file)
+
+
+    shutil.rmtree("docs/stable")
+    shutil.move("docs/stable_temp", "docs/stable")
+
+
+def update_stable_page(src_file, old_stable_version):
+    # parse YAML metadata and adjust the "redirect_from" field
+    doc = frontmatter.load(src_file)
+
+    redirect_from_field = doc.get("redirect_from")
+    if redirect_from_field:
+        redirect_from_field_to_archive = [
+            x.replace("docs/stable/", f"docs/{old_stable_version}/")
+            for x in redirect_from_field
+        ]
+        doc["redirect_from"] = redirect_from_field_to_archive
+
+    # replace link tags in the content
+    doc.content = doc.content.replace(
+        f"{{% link docs/stable/",
+        f"{{% link docs/{old_stable_version}/"
     )
-    print("Otherwise files are copied over as-is")
-    exit(1)
+    return frontmatter.dumps(doc)
 
-git_log_cmd = ['git', 'log', "--pretty=format:%H %cI"]
-git_ls_cmd = ['git', 'ls-tree', '--name-only']
-git_show_cmd = ['git', 'show']
-
-date = None
-confirm = True
-arguments = sys.argv
-for i in range(len(arguments)):
-    if arguments[i] == '--noconfirm':
-        confirm = False
-        del arguments[i]
-        i -= 1
-    elif arguments[i].startswith("--date="):
-        date = datetime.datetime.strptime(arguments[i].split('=')[1], '%Y-%m-%d')
-        date = datetime.datetime(date.year, date.month, date.day, 23, 59, 59)
-
-
-def execute_and_get_output(cmd):
-    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    stdout = proc.stdout.read()
-    stderr = proc.stderr.read()
-    proc.wait()
-    if proc.returncode != 0:
-        print(f"Command '{cmd}' failed!")
-        print(stdout)
-        print(stderr)
-        exit(1)
-    return stdout
-
-
-revision = None
-if date != None:
-    log_files = execute_and_get_output(git_log_cmd).decode('utf8').split('\n')
-    for line in log_files:
-        splits = line.split(' ')
-        parsed_date = datetime.datetime.strptime(splits[1], '%Y-%m-%dT%H:%M:%S%z')
-        rev_date = datetime.datetime(
-            parsed_date.year,
-            parsed_date.month,
-            parsed_date.day,
-            parsed_date.hour,
-            parsed_date.minute,
-            parsed_date.second,
-        )
-        if rev_date < date:
-            break
-        else:
-            revision = splits[0]
-    print(f"Formatting for {revision} (committed on {rev_date})")
-
-ignored_files = ['.DS_Store', 'archive', 'faq.md', 'why_duckdb.md']
-
-version = arguments[1]
-folder = os.path.join('docs', 'archive', version)
-
-print(
-    f"Archiving current docs for version \"{version}\" to path \"{folder}\". Remember to update _config.yml and /_data/versions.csv also."
-)
-if confirm:
-    result = input("Continue with archival (y/n)?\n")
-    if result != 'y':
-        print("Aborting.")
-        exit(0)
-
-
-def list_tree(source):
-    if revision == None:
-        return os.listdir(source)
-    else:
-        output = execute_and_get_output(git_ls_cmd + [revision, source + '/'])
-        output = output.decode('utf8').split('\n')
-        output = [os.path.basename(x) for x in output if len(x) > 0]
-        return output
-
-
-def copy_file(source_path, target_path, version):
-    print(f"{source_path} -> {target_path}")
-    if revision == None:
-        if source_path.endswith(".md"):
-            with open(source_path) as f, open(target_path, "w") as of:
-                # parse YAML metadata and adjust the "redirect_from" field
-                doc = frontmatter.load(f)
-
-                redirect_from_field = doc.get("redirect_from")
-                if redirect_from_field:
-                    redirect_from_field_to_archive = [
-                        x.replace("docs/", f"docs/archive/{version}/")
-                        for x in redirect_from_field
-                    ]
-                    doc["redirect_from"] = redirect_from_field_to_archive
-
-                doc.content = doc.content.replace(
-                    f"{{% link docs/",
-                    f"{{% link docs/archive/{version}/",
-                )
-
-                of.write(frontmatter.dumps(doc))
-        else:
-            shutil.copy(source_path, target_path)
-    else:
-        output = execute_and_get_output(git_show_cmd + [revision + ':' + source_path])
-        file_content = output
-        with open(target_path, 'wb+') as f:
-            f.write(file_content)
-
-
-def recursive_copy(source, target, version):
-    if not os.path.exists(target):
-        os.mkdir(target)
-    for fname in list_tree(source):
-        if fname in ignored_files:
-            continue
-        source_path = os.path.join(source, fname)
-        target_path = os.path.join(target, fname)
-        if os.path.isfile(source_path):
-            copy_file(source_path, target_path, version)
-        elif os.path.isdir(source_path):
-            recursive_copy(source_path, target_path, version)
-
-
-def archive_installation_page(version):
-    # get frontmatter of the proxy installation file (which includes the actual one)
-    with open(f"docs/installation/index.html") as current_installation_file:
-        current_installation_file_loaded = frontmatter.load(current_installation_file)
-        current_installation_file_loaded.content = ""
-
-    # adjust installation links to point to the old version in the archived page
-    with open(f"_includes/installation.html") as main_installation_file, open(
-        f"docs/archive/{version}/installation/index.html", "w"
-    ) as archived_installation_file:
-        installation_page = "\n" + main_installation_file.read()
-        installation_page = installation_page.replace(" (Latest Release)", "")
-        installation_page = installation_page.replace(
-            "{{ site.current_duckdb_version }}", version
-        )
-        # we leave the variable "{{ site.next_java_version }}" as is
-        # to allow the "GitHub main (Nightly Build)" to move with new versions
-
-        archived_installation_file.write(
-            frontmatter.dumps(current_installation_file_loaded)
-        )
-        archived_installation_file.write(installation_page)
-
-
-recursive_copy('docs', folder, version)
-copy_file(
-    '_data/menu_docs_stable.json',
-    '_data/menu_docs_%s.json' % (version.replace('.', ''),),
-    version,
-)
-archive_installation_page(version)
+
+# copy docs/stable to docs/<old_stable_version>
+# the directs should be expanded on with the version number
+def archive_stable(old_stable_version):
+    src = "docs/stable"
+    dst = f"docs/{old_stable_version}"
+
+    os.makedirs(dst, exist_ok=True)
+
+    for root, dirs, files in os.walk(src):
+        rel_path = os.path.relpath(root, src)
+        dest_dir = os.path.join(dst, rel_path)
+        os.makedirs(dest_dir, exist_ok=True)
+
+        for file in files:
+            src_file = os.path.join(root, file)
+            dst_file = os.path.join(dest_dir, file)
+
+            if src_file.endswith(".md"):
+                new_content = update_stable_page(src_file, old_stable_version)
+                print(new_content)
+                with open(dst_file, 'w') as f:
+                    f.write(new_content)
+            else:
+                shutil.copy2(src_file, dst_file)
+
+
+
+old_stable_version_no_dots = old_stable_version.replace(".", "")
+
+# copy_file("_data/menu_docs_stable.json", f"_data/menu_docs_{old_stable_version_no_dots}.json")
+archive_stable(old_stable_version)
+
+# copy_file("_data/menu_docs_preview.json", f"_data/menu_docs_stable.json")
+# archive_preview()