feat: extract bins

cmd/chisel/cmd_debug_check_release_archives.go

@@ -13,8 +13,8 @@ import (
 
 	"github.com/canonical/chisel/internal/archive"
 	"github.com/canonical/chisel/internal/cache"
-	"github.com/canonical/chisel/internal/deb"
 	"github.com/canonical/chisel/internal/setup"
+	"github.com/canonical/chisel/internal/tarball"
 )
 
 var shortCheckReleaseArchivesHelp = "Check the release's archives"
@@ -150,7 +150,7 @@ func computePathObservations(release *setup.Release, archives map[string]archive
 			if err != nil {
 				return nil, err
 			}
-			dataReader, err := deb.DataReader(pkgReader)
+			dataReader, err := tarball.DataReader(pkgReader, tarball.DebFormat)
 			if err != nil {
 				return nil, err
 			}

internal/archive/archive_test.go

@@ -1237,6 +1237,7 @@ func (s *S) testOpenArchiveArch(c *C, test realArchiveTest, arch string) {
 
 	err = tarball.Extract(pkg, &tarball.ExtractOptions{
 		Package:   test.pkg,
+		Format:    tarball.DebFormat,
 		TargetDir: extractDir,
 		Extract: map[string][]tarball.ExtractInfo{
 			fmt.Sprintf("/usr/share/doc/%s/copyright", test.pkg): {

internal/slicer/slicer.go

@@ -241,6 +241,7 @@ func Run(options *RunOptions) error {
 		}
 		err := tarball.Extract(reader, &tarball.ExtractOptions{
 			Package:   slice.Package,
+			Format:    tarball.DebFormat,
 			Extract:   extract[slice.Package],
 			TargetDir: targetDir,
 			Create:    create,

internal/tarball/extract.go

@@ -3,6 +3,7 @@ package tarball
 import (
 	"archive/tar"
 	"bytes"
+	"compress/gzip"
 	"fmt"
 	"io"
 	"io/fs"
@@ -12,13 +13,82 @@ import (
 	"strings"
 	"syscall"
 
-	"github.com/canonical/chisel/internal/deb"
+	"github.com/blakesmith/ar"
+	"github.com/klauspost/compress/zstd"
+	"github.com/ulikunitz/xz"
+
 	"github.com/canonical/chisel/internal/fsutil"
 	"github.com/canonical/chisel/internal/strdist"
 )
 
+// Format identifies the format of a package.
+type Format string
+
+const (
+	// DebFormat is the Debian package format: an ar archive holding a
+	// compressed data tarball.
+	DebFormat Format = "deb"
+	// BinFormat is the bin package format: a plain XZ-compressed tarball.
+	BinFormat Format = "bin"
+)
+
+// DataReader returns a reader over the uncompressed tar stream contained in
+// the given package, based on its format.
+func DataReader(pkgReader io.Reader, format Format) (io.ReadCloser, error) {
+	switch format {
+	case DebFormat:
+		return debDataReader(pkgReader)
+	case BinFormat:
+		xzReader, err := xz.NewReader(pkgReader)
+		if err != nil {
+			return nil, err
+		}
+		return io.NopCloser(xzReader), nil
+	}
+	return nil, fmt.Errorf("internal error: unsupported package format: %q", format)
+}
+
+// debDataReader returns a reader over the data tarball contained in the ar
+// file of a Debian package.
+func debDataReader(pkgReader io.Reader) (io.ReadCloser, error) {
+	arReader := ar.NewReader(pkgReader)
+	var dataReader io.ReadCloser
+	for dataReader == nil {
+		arHeader, err := arReader.Next()
+		if err == io.EOF {
+			return nil, fmt.Errorf("no data payload")
+		}
+		if err != nil {
+			return nil, err
+		}
+		switch arHeader.Name {
+		case "data.tar.gz":
+			gzipReader, err := gzip.NewReader(arReader)
+			if err != nil {
+				return nil, err
+			}
+			dataReader = gzipReader
+		case "data.tar.xz":
+			xzReader, err := xz.NewReader(arReader)
+			if err != nil {
+				return nil, err
+			}
+			dataReader = io.NopCloser(xzReader)
+		case "data.tar.zst":
+			zstdReader, err := zstd.NewReader(arReader)
+			if err != nil {
+				return nil, err
+			}
+			dataReader = zstdReader.IOReadCloser()
+		}
+	}
+
+	return dataReader, nil
+}
+
 type ExtractOptions struct {
 	Package   string
+	Format    Format
 	TargetDir string
 	Extract   map[string][]ExtractInfo
 	// Create can optionally be set to control the creation of extracted entries.
@@ -83,7 +153,7 @@ func Extract(pkgReader io.ReadSeeker, options *ExtractOptions) (err error) {
 }
 
 func extractData(pkgReader io.ReadSeeker, options *ExtractOptions) error {
-	dataReader, err := deb.DataReader(pkgReader)
+	dataReader, err := DataReader(pkgReader, options.Format)
 	if err != nil {
 		return err
 	}
@@ -117,8 +187,8 @@ func extractData(pkgReader io.ReadSeeker, options *ExtractOptions) error {
 	// create them with the permissions defined in the tarball.
 	//
 	// The assumption is that the tar entries of the parent directories appear
-	// before the entry for the file itself. This is the case for .deb files but
-	// not for all tarballs.
+	// before the entry for the file itself. This is the case for the tarballs
+	// produced by common packaging tools but not for all tarballs.
 	tarDirMode := make(map[string]fs.FileMode)
 	tarReader := tar.NewReader(dataReader)
 	for {
@@ -300,7 +370,7 @@ type extractHardLinkOptions struct {
 // extractHardLinks iterates through the tarball a second time to extract the
 // hard links that were not extracted in the first pass.
 func extractHardLinks(pkgReader io.ReadSeeker, opts *extractHardLinkOptions) error {
-	dataReader, err := deb.DataReader(pkgReader)
+	dataReader, err := DataReader(pkgReader, opts.Format)
 	if err != nil {
 		return err
 	}

internal/tarball/extract_test.go

@@ -496,6 +496,9 @@ func (s *S) TestExtract(c *C) {
 		options := test.options
 		options.Package = "test-package"
 		options.TargetDir = dir
+		if options.Format == "" {
+			options.Format = tarball.DebFormat
+		}
 		createdPaths := make(map[string]bool)
 		options.Create = func(_ []tarball.ExtractInfo, o *fsutil.CreateOptions) error {
 			relPath := filepath.Clean("/" + strings.TrimPrefix(o.Path, dir))
@@ -539,6 +542,7 @@ func (s *S) TestExtract(c *C) {
 var extractCreateCallbackTests = []struct {
 	summary string
 	pkgdata []byte
+	format  tarball.Format
 	options tarball.ExtractOptions
 	calls   map[string][]tarball.ExtractInfo
 }{{
@@ -605,6 +609,9 @@ func (s *S) TestExtractCreateCallback(c *C) {
 		options := test.options
 		options.Package = "test-package"
 		options.TargetDir = dir
+		if options.Format == "" {
+			options.Format = tarball.DebFormat
+		}
 		createExtractInfos := map[string][]tarball.ExtractInfo{}
 		options.Create = func(extractInfos []tarball.ExtractInfo, o *fsutil.CreateOptions) error {
 			if extractInfos == nil {