Merge pull request #64 from jiaqiluo/compress-crd-files

Author: jiaqiluo

.gitignore

@@ -4,4 +4,7 @@
 /dist
 *.swp
 .idea
-charts-build-scripts
+charts-build-scripts
+.DS_Store
+pkg/.DS_Store
+templates/.DS_Store

pkg/charts/additionalchart.go

@@ -44,6 +44,11 @@ func (c *AdditionalChart) ApplyMainChanges(pkgFs billy.Filesystem) error {
 	if err != nil {
 		return fmt.Errorf("Encountered error while trying to get the main chart's working directory: %s", err)
 	}
+	if c.CRDChartOptions.UseTarArchive {
+		if err := helm.ArchiveCRDs(pkgFs, mainChartWorkingDir, path.ChartCRDDir, c.WorkingDir, path.ChartExtraFileDir); err != nil {
+			return fmt.Errorf("encountered error while trying to bundle and compress CRD files from the main chart: %s", err)
+		}
+	}
 	if err := helm.CopyCRDsFromChart(pkgFs, mainChartWorkingDir, path.ChartCRDDir, c.WorkingDir, c.CRDChartOptions.CRDDirectory); err != nil {
 		return fmt.Errorf("Encountered error while trying to copy CRDs from %s to %s: %s", mainChartWorkingDir, c.WorkingDir, err)
 	}
@@ -66,12 +71,14 @@ func (c *AdditionalChart) RevertMainChanges(pkgFs billy.Filesystem) error {
 		return fmt.Errorf("Working directory %s has not been prepared yet", c.WorkingDir)
 	}
 	if c.CRDChartOptions == nil {
+		// return if the additional chart is not a CRD chart
 		return nil
 	}
 	mainChartWorkingDir, err := c.getMainChartWorkingDir(pkgFs)
 	if err != nil {
 		return fmt.Errorf("Encountered error while trying to get the main chart's working directory: %s", err)
 	}
+	// copy CRD files from packages/<package>/charts-crd/crd-manifest/ back to packages/<package>/charts/crds/
 	if err := helm.CopyCRDsFromChart(pkgFs, c.WorkingDir, c.CRDChartOptions.CRDDirectory, mainChartWorkingDir, path.ChartCRDDir); err != nil {
 		return fmt.Errorf("Encountered error while trying to copy CRDs from %s to %s: %s", c.WorkingDir, mainChartWorkingDir, err)
 	}

pkg/charts/package.go

@@ -39,6 +39,7 @@ func (p *Package) Prepare() error {
 		return fmt.Errorf("Encountered error while preparing main chart: %s", err)
 	}
 	if p.Chart.Upstream.IsWithinPackage() {
+		// in the case of local chart
 		for _, additionalChart := range p.AdditionalCharts {
 			exists, err := filesystem.PathExists(p.fs, additionalChart.WorkingDir)
 			if err != nil {

pkg/charts/parse.go

@@ -175,6 +175,7 @@ func GetAdditionalChartFromOptions(opt options.AdditionalChartOptions) (Addition
 			TemplateDirectory:           templateDirectory,
 			CRDDirectory:                crdDirectory,
 			AddCRDValidationToMainChart: opt.CRDChartOptions.AddCRDValidationToMainChart,
+			UseTarArchive:               opt.CRDChartOptions.UseTarArchive,
 		}
 	}
 	return a, nil

pkg/filesystem/filesystem.go

@@ -249,6 +249,52 @@ func UnarchiveTgz(fs billy.Filesystem, tgzPath, tgzSubdirectory, destPath string
 	return nil
 }
 
+// ArchiveDir archives a directory or a file into a tgz file and put it at destTgzPath which should end with .tgz
+func ArchiveDir(fs billy.Filesystem, srcPath, destTgzPath string) error {
+	if !strings.HasSuffix(destTgzPath, ".tgz") {
+		return fmt.Errorf("cannot archive %s to %s since the archive path does not end with '.tgz'", srcPath, destTgzPath)
+	}
+	tgzFile, err := fs.Create(destTgzPath)
+	if err != nil {
+		return err
+	}
+	defer tgzFile.Close()
+
+	gz := gzip.NewWriter(tgzFile)
+	defer gz.Close()
+
+	tarWriter := tar.NewWriter(gz)
+	defer tarWriter.Close()
+
+	return WalkDir(fs, srcPath, func(fs billy.Filesystem, path string, isDir bool) error {
+		info, err := fs.Stat(path)
+		if err != nil {
+			return err
+		}
+		header, err := tar.FileInfoHeader(info, info.Name())
+		if err != nil {
+			return err
+		}
+		// overwrite the name to be the full path to the file
+		header.Name = path
+		if err := tarWriter.WriteHeader(header); err != nil {
+			return err
+		}
+		// The directory structure is preserved, but there is no data to read from a directory
+		if isDir {
+			return nil
+		}
+		file, err := fs.Open(path)
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+
+		_, err = io.Copy(tarWriter, file)
+		return err
+	})
+}
+
 // RelativePathFunc is a function that is applied on a relative path within the given filesystem
 type RelativePathFunc func(fs billy.Filesystem, path string, isDir bool) error
 

pkg/helm/crds.go

@@ -49,3 +49,17 @@ func DeleteCRDsFromChart(fs billy.Filesystem, helmChartPath string) error {
 	}
 	return nil
 }
+
+// ArchiveCRDs bundles, compresses and saves the CRD files from the source to the destination
+func ArchiveCRDs(fs billy.Filesystem, srcHelmChartPath, srcCRDsDir, dstHelmChartPath, destCRDsDir string) error {
+	if err := filesystem.RemoveAll(fs, filepath.Join(dstHelmChartPath, destCRDsDir)); err != nil {
+		return err
+	}
+	if err := fs.MkdirAll(filepath.Join(dstHelmChartPath, destCRDsDir), os.ModePerm); err != nil {
+		return err
+	}
+	srcCRDsDirPath := filepath.Join(srcHelmChartPath, srcCRDsDir)
+	dstFilePath := filepath.Join(dstHelmChartPath, destCRDsDir, fmt.Sprintf("%s.tgz", "crd-manifest"))
+	logrus.Infof("Compressing CRDs from %s to %s", srcCRDsDirPath, dstFilePath)
+	return filesystem.ArchiveDir(fs, srcCRDsDirPath, dstFilePath)
+}

pkg/options/additionalchart.go

@@ -18,4 +18,6 @@ type CRDChartOptions struct {
 	CRDDirectory string `yaml:"crdDirectory" default:"templates"`
 	// Whether to add a validation file to your main chart to check that CRDs exist
 	AddCRDValidationToMainChart bool `yaml:"addCRDValidationToMainChart"`
+	// UseTarArchive indicates whether to bundle and compress CRD files into a tgz file
+	UseTarArchive bool `yaml:"useTarArchive"`
 }

pkg/path/path.go

@@ -37,6 +37,8 @@ const (
 
 	// ChartCRDDir represents the directory that we expect to contain CRDs within the chart
 	ChartCRDDir = "crds"
+	// ChartExtraFileDir represents the directory that contains non-YAML files
+	ChartExtraFileDir = "files"
 	// ChartValidateInstallCRDFile is the path to the file pushed to upstream that validates the existence of CRDs in the chart
 	ChartValidateInstallCRDFile = "templates/validate-install-crd.yaml"
 )