From f91d4498df94314ae797908bee77b0cc8e437e8e Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 30 Jun 2026 09:15:13 +0200 Subject: [PATCH 01/15] feat: extract bins --- internal/archive/archive_test.go | 2 ++ internal/slicer/slicer.go | 2 ++ internal/tarball/extract.go | 13 ++++++++--- internal/tarball/extract_test.go | 17 +++++++++++++++ internal/tarball/xz.go | 15 +++++++++++++ internal/testutil/pkgdata.go | 37 ++++++++++++++++++++++++++++++++ 6 files changed, 83 insertions(+), 3 deletions(-) create mode 100644 internal/tarball/xz.go diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index 90e3f0047..fcabc2a25 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -17,6 +17,7 @@ import ( "strings" "github.com/canonical/chisel/internal/archive" + "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/archive/testarchive" "github.com/canonical/chisel/internal/tarball" "github.com/canonical/chisel/internal/testutil" @@ -986,6 +987,7 @@ func (s *S) testOpenArchiveArch(c *C, test realArchiveTest, arch string) { err = tarball.Extract(pkg, &tarball.ExtractOptions{ Package: test.pkg, TargetDir: extractDir, + OpenData: deb.DataReader, Extract: map[string][]tarball.ExtractInfo{ fmt.Sprintf("/usr/share/doc/%s/copyright", test.pkg): { {Path: "/copyright"}, diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 6f4783b88..00ed3ee37 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -16,6 +16,7 @@ import ( "github.com/klauspost/compress/zstd" "github.com/canonical/chisel/internal/archive" + "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/fsutil" "github.com/canonical/chisel/internal/manifestutil" "github.com/canonical/chisel/internal/scripts" @@ -243,6 +244,7 @@ func Run(options *RunOptions) error { Package: slice.Package, Extract: extract[slice.Package], TargetDir: targetDir, + OpenData: deb.DataReader, Create: create, }) reader.Close() diff --git a/internal/tarball/extract.go b/internal/tarball/extract.go index 40db2cbac..ce1fda5b2 100644 --- a/internal/tarball/extract.go +++ b/internal/tarball/extract.go @@ -12,7 +12,6 @@ import ( "strings" "syscall" - "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/fsutil" "github.com/canonical/chisel/internal/strdist" ) @@ -21,6 +20,11 @@ type ExtractOptions struct { Package string TargetDir string Extract map[string][]ExtractInfo + // OpenData opens the uncompressed tar data stream of the package. It + // abstracts over the package format (e.g. a deb archive opener or a plain + // tarball opener), allowing Extract to operate on any package whose data + // payload is a tar stream. + OpenData func(io.ReadSeeker) (io.ReadCloser, error) // Create can optionally be set to control the creation of extracted entries. // extractInfos is set to the matching entries in Extract, and is nil in cases where // the created entry is implicit and unlisted (for example, parent directories). @@ -35,6 +39,9 @@ type ExtractInfo struct { } func getValidOptions(options *ExtractOptions) (*ExtractOptions, error) { + if options.OpenData == nil { + return nil, fmt.Errorf("internal error: ExtractOptions.OpenData is unset") + } for extractPath, extractInfos := range options.Extract { isGlob := strings.ContainsAny(extractPath, "*?") if isGlob { @@ -83,7 +90,7 @@ func Extract(pkgReader io.ReadSeeker, options *ExtractOptions) (err error) { } func extractData(pkgReader io.ReadSeeker, options *ExtractOptions) error { - dataReader, err := deb.DataReader(pkgReader) + dataReader, err := options.OpenData(pkgReader) if err != nil { return err } @@ -300,7 +307,7 @@ type extractHardLinkOptions struct { // extractHardLinks iterates through the tarball a second time to extract the // hard links that were not extracted in the first pass. func extractHardLinks(pkgReader io.ReadSeeker, opts *extractHardLinkOptions) error { - dataReader, err := deb.DataReader(pkgReader) + dataReader, err := opts.OpenData(pkgReader) if err != nil { return err } diff --git a/internal/tarball/extract_test.go b/internal/tarball/extract_test.go index fd5eb6147..f855d9bc4 100644 --- a/internal/tarball/extract_test.go +++ b/internal/tarball/extract_test.go @@ -10,6 +10,7 @@ import ( . "gopkg.in/check.v1" + "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/fsutil" "github.com/canonical/chisel/internal/tarball" "github.com/canonical/chisel/internal/testutil" @@ -496,6 +497,8 @@ func (s *S) TestExtract(c *C) { options := test.options options.Package = "test-package" options.TargetDir = dir + // The test fixtures are .deb archives, so use the deb data opener. + options.OpenData = deb.DataReader createdPaths := make(map[string]bool) options.Create = func(_ []tarball.ExtractInfo, o *fsutil.CreateOptions) error { relPath := filepath.Clean("/" + strings.TrimPrefix(o.Path, dir)) @@ -605,6 +608,8 @@ func (s *S) TestExtractCreateCallback(c *C) { options := test.options options.Package = "test-package" options.TargetDir = dir + // The test fixtures are .deb archives, so use the deb data opener. + options.OpenData = deb.DataReader createExtractInfos := map[string][]tarball.ExtractInfo{} options.Create = func(extractInfos []tarball.ExtractInfo, o *fsutil.CreateOptions) error { if extractInfos == nil { @@ -628,3 +633,15 @@ func (s *S) TestExtractCreateCallback(c *C) { c.Assert(createExtractInfos, DeepEquals, test.calls) } } + +func (s *S) TestExtractMissingOpenData(c *C) { + options := tarball.ExtractOptions{ + Package: "test-package", + TargetDir: c.MkDir(), + Extract: map[string][]tarball.ExtractInfo{ + "/dir/file": {{Path: "/dir/file"}}, + }, + } + err := tarball.Extract(bytes.NewReader(testutil.PackageData["test-package"]), &options) + c.Assert(err, ErrorMatches, `cannot extract from package "test-package": internal error: ExtractOptions.OpenData is unset`) +} diff --git a/internal/tarball/xz.go b/internal/tarball/xz.go new file mode 100644 index 000000000..700215a09 --- /dev/null +++ b/internal/tarball/xz.go @@ -0,0 +1,15 @@ +package tarball + +import ( + "io" + + "github.com/ulikunitz/xz" +) + +func XZDataReader(pkgReader io.ReadSeeker) (io.ReadCloser, error) { + xzReader, err := xz.NewReader(pkgReader) + if err != nil { + return nil, err + } + return io.NopCloser(xzReader), nil +} diff --git a/internal/testutil/pkgdata.go b/internal/testutil/pkgdata.go index f901873d3..8fb8fb8b2 100644 --- a/internal/testutil/pkgdata.go +++ b/internal/testutil/pkgdata.go @@ -8,6 +8,7 @@ import ( "github.com/blakesmith/ar" "github.com/klauspost/compress/zstd" + "github.com/ulikunitz/xz" ) var PackageData = map[string][]byte{} @@ -160,6 +161,42 @@ func MustMakeDeb(entries []TarEntry) []byte { return data } +// compressBytesXz compresses the input using XZ, the compression format used by +// store (bin) packages. +func compressBytesXz(input []byte) ([]byte, error) { + var buf bytes.Buffer + writer, err := xz.NewWriter(&buf) + if err != nil { + return nil, err + } + if _, err = writer.Write(input); err != nil { + return nil, err + } + if err = writer.Close(); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// MakeBin builds a store (bin) package from the given tar entries: an +// XZ-compressed plain tarball. Unlike MakeDeb, there is no ar container. +func MakeBin(entries []TarEntry) ([]byte, error) { + tarData, err := makeTar(entries) + if err != nil { + return nil, err + } + return compressBytesXz(tarData) +} + +// MustMakeBin is the panicking variant of MakeBin. +func MustMakeBin(entries []TarEntry) []byte { + data, err := MakeBin(entries) + if err != nil { + panic(err) + } + return data +} + // Reg is a shortcut for creating a regular file TarEntry structure (with // tar.Typeflag set tar.TypeReg). Reg stands for "REGular file". func Reg(mode int64, path, content string) TarEntry { From 166344644b84dbb1058028913eac09dbda4c45a4 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 29 Jul 2026 11:54:44 +0200 Subject: [PATCH 02/15] refactor: rework approach --- .../cmd_debug_check_release_archives.go | 2 +- internal/archive/archive_test.go | 5 +- internal/deb/extract.go | 7 ++- internal/slicer/slicer.go | 3 +- internal/tarball/extract.go | 58 ++++++++++++++----- internal/tarball/extract_test.go | 16 +++-- internal/tarball/xz.go | 15 ----- 7 files changed, 57 insertions(+), 49 deletions(-) delete mode 100644 internal/tarball/xz.go diff --git a/cmd/chisel/cmd_debug_check_release_archives.go b/cmd/chisel/cmd_debug_check_release_archives.go index c9c6270e4..29a8189a8 100644 --- a/cmd/chisel/cmd_debug_check_release_archives.go +++ b/cmd/chisel/cmd_debug_check_release_archives.go @@ -150,7 +150,7 @@ func computePathObservations(release *setup.Release, archives map[string]archive if err != nil { return nil, err } - dataReader, err := deb.DataReader(pkgReader) + dataReader, err := deb.OpenTar(pkgReader) if err != nil { return nil, err } diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index fcabc2a25..efde6658c 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -17,8 +17,8 @@ import ( "strings" "github.com/canonical/chisel/internal/archive" - "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/archive/testarchive" + "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/tarball" "github.com/canonical/chisel/internal/testutil" ) @@ -984,10 +984,9 @@ func (s *S) testOpenArchiveArch(c *C, test realArchiveTest, arch string) { c.Assert(info.Name, DeepEquals, test.pkg) c.Assert(info.Arch, DeepEquals, arch) - err = tarball.Extract(pkg, &tarball.ExtractOptions{ + err = tarball.Extract(pkg, deb.OpenTar, &tarball.ExtractOptions{ Package: test.pkg, TargetDir: extractDir, - OpenData: deb.DataReader, Extract: map[string][]tarball.ExtractInfo{ fmt.Sprintf("/usr/share/doc/%s/copyright", test.pkg): { {Path: "/copyright"}, diff --git a/internal/deb/extract.go b/internal/deb/extract.go index bf543c2d8..49c0ba13b 100644 --- a/internal/deb/extract.go +++ b/internal/deb/extract.go @@ -10,9 +10,10 @@ import ( "github.com/ulikunitz/xz" ) -// DataReader takes a Reader for the ar file belonging to a Debian package and -// returns a Reader to the inner tarball. -func DataReader(pkgReader io.ReadSeeker) (io.ReadCloser, error) { +// OpenTar takes a Reader for the ar file belonging to a Debian package and +// returns a Reader to the uncompressed inner tarball. It implements +// tarball.OpenTarFunc. +func OpenTar(pkgReader io.ReadSeeker) (io.ReadCloser, error) { arReader := ar.NewReader(pkgReader) var dataReader io.ReadCloser for dataReader == nil { diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 00ed3ee37..8fcc1c0e2 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -240,11 +240,10 @@ func Run(options *RunOptions) error { if reader == nil { continue } - err := tarball.Extract(reader, &tarball.ExtractOptions{ + err := tarball.Extract(reader, deb.OpenTar, &tarball.ExtractOptions{ Package: slice.Package, Extract: extract[slice.Package], TargetDir: targetDir, - OpenData: deb.DataReader, Create: create, }) reader.Close() diff --git a/internal/tarball/extract.go b/internal/tarball/extract.go index ce1fda5b2..bc78534ac 100644 --- a/internal/tarball/extract.go +++ b/internal/tarball/extract.go @@ -12,19 +12,41 @@ import ( "strings" "syscall" + "github.com/ulikunitz/xz" + "github.com/canonical/chisel/internal/fsutil" "github.com/canonical/chisel/internal/strdist" ) +// OpenTarFunc opens the uncompressed tar stream carried by a package, hiding +// the package format from Extract. An implementation may unwrap a container +// before decompressing (see deb.OpenTar) or decompress the package itself (see +// OpenXZ). +// +// Implementations must satisfy the following contract: +// +// - They must read pkgReader from its current offset, which is always the +// start of the package. +// - They must be stateless. Extract may call the same function more than once +// for the same package, rewinding pkgReader to the start beforehand, and +// each call must yield the complete tar stream again. +// - The caller closes the returned reader. +type OpenTarFunc func(pkgReader io.ReadSeeker) (io.ReadCloser, error) + +// OpenXZ opens a package which is a plain XZ-compressed tarball, such as a +// store (bin) package. It implements OpenTarFunc. +func OpenXZ(pkgReader io.ReadSeeker) (io.ReadCloser, error) { + xzReader, err := xz.NewReader(pkgReader) + if err != nil { + return nil, err + } + return io.NopCloser(xzReader), nil +} + type ExtractOptions struct { Package string TargetDir string Extract map[string][]ExtractInfo - // OpenData opens the uncompressed tar data stream of the package. It - // abstracts over the package format (e.g. a deb archive opener or a plain - // tarball opener), allowing Extract to operate on any package whose data - // payload is a tar stream. - OpenData func(io.ReadSeeker) (io.ReadCloser, error) // Create can optionally be set to control the creation of extracted entries. // extractInfos is set to the matching entries in Extract, and is nil in cases where // the created entry is implicit and unlisted (for example, parent directories). @@ -39,9 +61,6 @@ type ExtractInfo struct { } func getValidOptions(options *ExtractOptions) (*ExtractOptions, error) { - if options.OpenData == nil { - return nil, fmt.Errorf("internal error: ExtractOptions.OpenData is unset") - } for extractPath, extractInfos := range options.Extract { isGlob := strings.ContainsAny(extractPath, "*?") if isGlob { @@ -65,7 +84,10 @@ func getValidOptions(options *ExtractOptions) (*ExtractOptions, error) { return options, nil } -func Extract(pkgReader io.ReadSeeker, options *ExtractOptions) (err error) { +// Extract extracts from pkgReader the entries listed in options.Extract. +// openTar opens the tar stream carried by the package and must not be nil; see +// OpenTarFunc for the contract it must satisfy. +func Extract(pkgReader io.ReadSeeker, openTar OpenTarFunc, options *ExtractOptions) (err error) { defer func() { if err != nil { err = fmt.Errorf("cannot extract from package %q: %w", options.Package, err) @@ -74,6 +96,10 @@ func Extract(pkgReader io.ReadSeeker, options *ExtractOptions) (err error) { logf("Extracting files from package %q...", options.Package) + if openTar == nil { + return fmt.Errorf("internal error: no tar opener provided") + } + validOpts, err := getValidOptions(options) if err != nil { return err @@ -86,11 +112,11 @@ func Extract(pkgReader io.ReadSeeker, options *ExtractOptions) (err error) { return err } - return extractData(pkgReader, validOpts) + return extractData(pkgReader, openTar, validOpts) } -func extractData(pkgReader io.ReadSeeker, options *ExtractOptions) error { - dataReader, err := options.OpenData(pkgReader) +func extractData(pkgReader io.ReadSeeker, openTar OpenTarFunc, options *ExtractOptions) error { + dataReader, err := openTar(pkgReader) if err != nil { return err } @@ -172,7 +198,7 @@ func extractData(pkgReader io.ReadSeeker, options *ExtractOptions) error { } var contentCache []byte - var contentIsCached = len(targetPaths) > 1 && !sourceIsDir + contentIsCached := len(targetPaths) > 1 && !sourceIsDir if contentIsCached { // Read and cache the content so it may be reused. // As an alternative, to avoid having an entire file in @@ -272,7 +298,7 @@ func extractData(pkgReader io.ReadSeeker, options *ExtractOptions) error { if err != nil { return err } - err = extractHardLinks(pkgReader, extractHardLinkOptions) + err = extractHardLinks(pkgReader, openTar, extractHardLinkOptions) if err != nil { return err } @@ -306,8 +332,8 @@ type extractHardLinkOptions struct { // extractHardLinks iterates through the tarball a second time to extract the // hard links that were not extracted in the first pass. -func extractHardLinks(pkgReader io.ReadSeeker, opts *extractHardLinkOptions) error { - dataReader, err := opts.OpenData(pkgReader) +func extractHardLinks(pkgReader io.ReadSeeker, openTar OpenTarFunc, opts *extractHardLinkOptions) error { + dataReader, err := openTar(pkgReader) if err != nil { return err } diff --git a/internal/tarball/extract_test.go b/internal/tarball/extract_test.go index f855d9bc4..a2246bbec 100644 --- a/internal/tarball/extract_test.go +++ b/internal/tarball/extract_test.go @@ -497,8 +497,6 @@ func (s *S) TestExtract(c *C) { options := test.options options.Package = "test-package" options.TargetDir = dir - // The test fixtures are .deb archives, so use the deb data opener. - options.OpenData = deb.DataReader createdPaths := make(map[string]bool) options.Create = func(_ []tarball.ExtractInfo, o *fsutil.CreateOptions) error { relPath := filepath.Clean("/" + strings.TrimPrefix(o.Path, dir)) @@ -514,7 +512,8 @@ func (s *S) TestExtract(c *C) { test.hackopt(c, &options) } - err := tarball.Extract(bytes.NewReader(test.pkgdata), &options) + // The test fixtures are .deb archives, so use the deb tar opener. + err := tarball.Extract(bytes.NewReader(test.pkgdata), deb.OpenTar, &options) if test.error != "" { c.Assert(err, ErrorMatches, test.error) continue @@ -608,8 +607,6 @@ func (s *S) TestExtractCreateCallback(c *C) { options := test.options options.Package = "test-package" options.TargetDir = dir - // The test fixtures are .deb archives, so use the deb data opener. - options.OpenData = deb.DataReader createExtractInfos := map[string][]tarball.ExtractInfo{} options.Create = func(extractInfos []tarball.ExtractInfo, o *fsutil.CreateOptions) error { if extractInfos == nil { @@ -627,14 +624,15 @@ func (s *S) TestExtractCreateCallback(c *C) { return nil } - err := tarball.Extract(bytes.NewReader(test.pkgdata), &options) + // The test fixtures are .deb archives, so use the deb tar opener. + err := tarball.Extract(bytes.NewReader(test.pkgdata), deb.OpenTar, &options) c.Assert(err, IsNil) c.Assert(createExtractInfos, DeepEquals, test.calls) } } -func (s *S) TestExtractMissingOpenData(c *C) { +func (s *S) TestExtractMissingOpenTar(c *C) { options := tarball.ExtractOptions{ Package: "test-package", TargetDir: c.MkDir(), @@ -642,6 +640,6 @@ func (s *S) TestExtractMissingOpenData(c *C) { "/dir/file": {{Path: "/dir/file"}}, }, } - err := tarball.Extract(bytes.NewReader(testutil.PackageData["test-package"]), &options) - c.Assert(err, ErrorMatches, `cannot extract from package "test-package": internal error: ExtractOptions.OpenData is unset`) + err := tarball.Extract(bytes.NewReader(testutil.PackageData["test-package"]), nil, &options) + c.Assert(err, ErrorMatches, `cannot extract from package "test-package": internal error: no tar opener provided`) } diff --git a/internal/tarball/xz.go b/internal/tarball/xz.go deleted file mode 100644 index 700215a09..000000000 --- a/internal/tarball/xz.go +++ /dev/null @@ -1,15 +0,0 @@ -package tarball - -import ( - "io" - - "github.com/ulikunitz/xz" -) - -func XZDataReader(pkgReader io.ReadSeeker) (io.ReadCloser, error) { - xzReader, err := xz.NewReader(pkgReader) - if err != nil { - return nil, err - } - return io.NopCloser(xzReader), nil -} From 3435cee29c65c8eda6477bc253dd4f2a1845d625 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 29 Jul 2026 13:23:15 +0200 Subject: [PATCH 03/15] refactor: refining --- internal/deb/extract.go | 5 ++--- internal/tarball/extract.go | 20 ++++++------------- internal/testutil/pkgdata.go | 37 ------------------------------------ 3 files changed, 8 insertions(+), 54 deletions(-) diff --git a/internal/deb/extract.go b/internal/deb/extract.go index 49c0ba13b..d25f1f883 100644 --- a/internal/deb/extract.go +++ b/internal/deb/extract.go @@ -11,9 +11,8 @@ import ( ) // OpenTar takes a Reader for the ar file belonging to a Debian package and -// returns a Reader to the uncompressed inner tarball. It implements -// tarball.OpenTarFunc. -func OpenTar(pkgReader io.ReadSeeker) (io.ReadCloser, error) { +// returns a Reader to the uncompressed inner tarball. +func OpenTar(pkgReader io.Reader) (io.ReadCloser, error) { arReader := ar.NewReader(pkgReader) var dataReader io.ReadCloser for dataReader == nil { diff --git a/internal/tarball/extract.go b/internal/tarball/extract.go index bc78534ac..26c8ec6a9 100644 --- a/internal/tarball/extract.go +++ b/internal/tarball/extract.go @@ -23,19 +23,14 @@ import ( // before decompressing (see deb.OpenTar) or decompress the package itself (see // OpenXZ). // -// Implementations must satisfy the following contract: -// -// - They must read pkgReader from its current offset, which is always the -// start of the package. -// - They must be stateless. Extract may call the same function more than once -// for the same package, rewinding pkgReader to the start beforehand, and -// each call must yield the complete tar stream again. -// - The caller closes the returned reader. -type OpenTarFunc func(pkgReader io.ReadSeeker) (io.ReadCloser, error) +// Extract reads pkgReader from the start of the package, closes the returned +// reader, and may open the same package more than once, rewinding pkgReader +// beforehand. +type OpenTarFunc func(pkgReader io.Reader) (io.ReadCloser, error) // OpenXZ opens a package which is a plain XZ-compressed tarball, such as a -// store (bin) package. It implements OpenTarFunc. -func OpenXZ(pkgReader io.ReadSeeker) (io.ReadCloser, error) { +// store (bin) package. +func OpenXZ(pkgReader io.Reader) (io.ReadCloser, error) { xzReader, err := xz.NewReader(pkgReader) if err != nil { return nil, err @@ -84,9 +79,6 @@ func getValidOptions(options *ExtractOptions) (*ExtractOptions, error) { return options, nil } -// Extract extracts from pkgReader the entries listed in options.Extract. -// openTar opens the tar stream carried by the package and must not be nil; see -// OpenTarFunc for the contract it must satisfy. func Extract(pkgReader io.ReadSeeker, openTar OpenTarFunc, options *ExtractOptions) (err error) { defer func() { if err != nil { diff --git a/internal/testutil/pkgdata.go b/internal/testutil/pkgdata.go index 8fb8fb8b2..f901873d3 100644 --- a/internal/testutil/pkgdata.go +++ b/internal/testutil/pkgdata.go @@ -8,7 +8,6 @@ import ( "github.com/blakesmith/ar" "github.com/klauspost/compress/zstd" - "github.com/ulikunitz/xz" ) var PackageData = map[string][]byte{} @@ -161,42 +160,6 @@ func MustMakeDeb(entries []TarEntry) []byte { return data } -// compressBytesXz compresses the input using XZ, the compression format used by -// store (bin) packages. -func compressBytesXz(input []byte) ([]byte, error) { - var buf bytes.Buffer - writer, err := xz.NewWriter(&buf) - if err != nil { - return nil, err - } - if _, err = writer.Write(input); err != nil { - return nil, err - } - if err = writer.Close(); err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -// MakeBin builds a store (bin) package from the given tar entries: an -// XZ-compressed plain tarball. Unlike MakeDeb, there is no ar container. -func MakeBin(entries []TarEntry) ([]byte, error) { - tarData, err := makeTar(entries) - if err != nil { - return nil, err - } - return compressBytesXz(tarData) -} - -// MustMakeBin is the panicking variant of MakeBin. -func MustMakeBin(entries []TarEntry) []byte { - data, err := MakeBin(entries) - if err != nil { - panic(err) - } - return data -} - // Reg is a shortcut for creating a regular file TarEntry structure (with // tar.Typeflag set tar.TypeReg). Reg stands for "REGular file". func Reg(mode int64, path, content string) TarEntry { From 09a01bd80b8ef36a30af041c46a2a62dc64c212e Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 29 Jul 2026 13:30:16 +0200 Subject: [PATCH 04/15] refactor: refining --- internal/tarball/extract.go | 6 +----- internal/tarball/extract_test.go | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/internal/tarball/extract.go b/internal/tarball/extract.go index 26c8ec6a9..28d05be9e 100644 --- a/internal/tarball/extract.go +++ b/internal/tarball/extract.go @@ -22,10 +22,6 @@ import ( // the package format from Extract. An implementation may unwrap a container // before decompressing (see deb.OpenTar) or decompress the package itself (see // OpenXZ). -// -// Extract reads pkgReader from the start of the package, closes the returned -// reader, and may open the same package more than once, rewinding pkgReader -// beforehand. type OpenTarFunc func(pkgReader io.Reader) (io.ReadCloser, error) // OpenXZ opens a package which is a plain XZ-compressed tarball, such as a @@ -190,7 +186,7 @@ func extractData(pkgReader io.ReadSeeker, openTar OpenTarFunc, options *ExtractO } var contentCache []byte - contentIsCached := len(targetPaths) > 1 && !sourceIsDir + var contentIsCached = len(targetPaths) > 1 && !sourceIsDir if contentIsCached { // Read and cache the content so it may be reused. // As an alternative, to avoid having an entire file in diff --git a/internal/tarball/extract_test.go b/internal/tarball/extract_test.go index a2246bbec..9c5fe3523 100644 --- a/internal/tarball/extract_test.go +++ b/internal/tarball/extract_test.go @@ -19,6 +19,8 @@ import ( type extractTest struct { summary string pkgdata []byte + // openTar must match the format of pkgdata. It defaults to deb.OpenTar. + openTar tarball.OpenTarFunc options tarball.ExtractOptions hackopt func(c *C, o *tarball.ExtractOptions) result map[string]string @@ -512,8 +514,11 @@ func (s *S) TestExtract(c *C) { test.hackopt(c, &options) } - // The test fixtures are .deb archives, so use the deb tar opener. - err := tarball.Extract(bytes.NewReader(test.pkgdata), deb.OpenTar, &options) + openTar := test.openTar + if openTar == nil { + openTar = deb.OpenTar + } + err := tarball.Extract(bytes.NewReader(test.pkgdata), openTar, &options) if test.error != "" { c.Assert(err, ErrorMatches, test.error) continue @@ -541,6 +546,8 @@ func (s *S) TestExtract(c *C) { var extractCreateCallbackTests = []struct { summary string pkgdata []byte + // openTar must match the format of pkgdata. It defaults to deb.OpenTar. + openTar tarball.OpenTarFunc options tarball.ExtractOptions calls map[string][]tarball.ExtractInfo }{{ @@ -624,8 +631,11 @@ func (s *S) TestExtractCreateCallback(c *C) { return nil } - // The test fixtures are .deb archives, so use the deb tar opener. - err := tarball.Extract(bytes.NewReader(test.pkgdata), deb.OpenTar, &options) + openTar := test.openTar + if openTar == nil { + openTar = deb.OpenTar + } + err := tarball.Extract(bytes.NewReader(test.pkgdata), openTar, &options) c.Assert(err, IsNil) c.Assert(createExtractInfos, DeepEquals, test.calls) From 8d87fae3b1ea212fe44dedc2188d1e452c52646f Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 19 Aug 2026 11:02:15 +0200 Subject: [PATCH 05/15] docs: clean comments --- internal/tarball/extract.go | 2 +- internal/tarball/extract_test.go | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/internal/tarball/extract.go b/internal/tarball/extract.go index 28d05be9e..de014be7e 100644 --- a/internal/tarball/extract.go +++ b/internal/tarball/extract.go @@ -25,7 +25,7 @@ import ( type OpenTarFunc func(pkgReader io.Reader) (io.ReadCloser, error) // OpenXZ opens a package which is a plain XZ-compressed tarball, such as a -// store (bin) package. +// bin package. func OpenXZ(pkgReader io.Reader) (io.ReadCloser, error) { xzReader, err := xz.NewReader(pkgReader) if err != nil { diff --git a/internal/tarball/extract_test.go b/internal/tarball/extract_test.go index 9c5fe3523..324936095 100644 --- a/internal/tarball/extract_test.go +++ b/internal/tarball/extract_test.go @@ -19,7 +19,6 @@ import ( type extractTest struct { summary string pkgdata []byte - // openTar must match the format of pkgdata. It defaults to deb.OpenTar. openTar tarball.OpenTarFunc options tarball.ExtractOptions hackopt func(c *C, o *tarball.ExtractOptions) @@ -546,7 +545,6 @@ func (s *S) TestExtract(c *C) { var extractCreateCallbackTests = []struct { summary string pkgdata []byte - // openTar must match the format of pkgdata. It defaults to deb.OpenTar. openTar tarball.OpenTarFunc options tarball.ExtractOptions calls map[string][]tarball.ExtractInfo From 1a38ccdf8f5930bb2da03a8479165827314b76be Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 1 Sep 2026 09:33:48 +0200 Subject: [PATCH 06/15] docs: remove mentions of packages in tarball pkg --- internal/tarball/extract.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/internal/tarball/extract.go b/internal/tarball/extract.go index de014be7e..d98ba8bb8 100644 --- a/internal/tarball/extract.go +++ b/internal/tarball/extract.go @@ -18,14 +18,13 @@ import ( "github.com/canonical/chisel/internal/strdist" ) -// OpenTarFunc opens the uncompressed tar stream carried by a package, hiding -// the package format from Extract. An implementation may unwrap a container -// before decompressing (see deb.OpenTar) or decompress the package itself (see -// OpenXZ). +// OpenTarFunc returns a reader over the uncompressed tar stream contained in +// its input, hiding the container and compression details from Extract. An +// implementation may unwrap a container before decompressing (see deb.OpenTar) +// or decompress the input directly (see OpenXZ). type OpenTarFunc func(pkgReader io.Reader) (io.ReadCloser, error) -// OpenXZ opens a package which is a plain XZ-compressed tarball, such as a -// bin package. +// OpenXZ opens a plain XZ-compressed tarball. func OpenXZ(pkgReader io.Reader) (io.ReadCloser, error) { xzReader, err := xz.NewReader(pkgReader) if err != nil { @@ -138,8 +137,8 @@ func extractData(pkgReader io.ReadSeeker, openTar OpenTarFunc, options *ExtractO // create them with the permissions defined in the tarball. // // The assumption is that the tar entries of the parent directories appear - // before the entry for the file itself. This is the case for .deb files but - // not for all tarballs. + // before the entry for the file itself. This is the case for the tarballs + // produced by common packaging tools but not for all tarballs. tarDirMode := make(map[string]fs.FileMode) tarReader := tar.NewReader(dataReader) for { @@ -382,7 +381,7 @@ func extractHardLinks(pkgReader io.ReadSeeker, openTar OpenTarFunc, opts *extrac } // If there are pending links, that means the link targets do not come from - // this package. + // this tarball. if len(opts.pendingLinks) > 0 { var targets []string for target := range opts.pendingLinks { From ee8732aad0f842ad513f45ea9a6db29e3dcdfef7 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Thu, 3 Sep 2026 09:30:48 +0200 Subject: [PATCH 07/15] style: refine naming and docs --- internal/tarball/extract.go | 32 +++++++++++++++----------------- internal/tarball/extract_test.go | 4 ++-- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/internal/tarball/extract.go b/internal/tarball/extract.go index d98ba8bb8..597e8ab96 100644 --- a/internal/tarball/extract.go +++ b/internal/tarball/extract.go @@ -18,14 +18,12 @@ import ( "github.com/canonical/chisel/internal/strdist" ) -// OpenTarFunc returns a reader over the uncompressed tar stream contained in -// its input, hiding the container and compression details from Extract. An -// implementation may unwrap a container before decompressing (see deb.OpenTar) -// or decompress the input directly (see OpenXZ). -type OpenTarFunc func(pkgReader io.Reader) (io.ReadCloser, error) - -// OpenXZ opens a plain XZ-compressed tarball. -func OpenXZ(pkgReader io.Reader) (io.ReadCloser, error) { +// TarOpener returns a reader over the uncompressed tar stream contained in +// its input, hiding the container and compression details from Extract. +type TarOpener func(pkgReader io.Reader) (io.ReadCloser, error) + +// OpenXZTar returns a reader over the decompressed XZ stream. +func OpenXZTar(pkgReader io.Reader) (io.ReadCloser, error) { xzReader, err := xz.NewReader(pkgReader) if err != nil { return nil, err @@ -74,7 +72,7 @@ func getValidOptions(options *ExtractOptions) (*ExtractOptions, error) { return options, nil } -func Extract(pkgReader io.ReadSeeker, openTar OpenTarFunc, options *ExtractOptions) (err error) { +func Extract(pkgReader io.ReadSeeker, opener TarOpener, options *ExtractOptions) (err error) { defer func() { if err != nil { err = fmt.Errorf("cannot extract from package %q: %w", options.Package, err) @@ -83,7 +81,7 @@ func Extract(pkgReader io.ReadSeeker, openTar OpenTarFunc, options *ExtractOptio logf("Extracting files from package %q...", options.Package) - if openTar == nil { + if opener == nil { return fmt.Errorf("internal error: no tar opener provided") } @@ -99,11 +97,11 @@ func Extract(pkgReader io.ReadSeeker, openTar OpenTarFunc, options *ExtractOptio return err } - return extractData(pkgReader, openTar, validOpts) + return extractData(pkgReader, opener, validOpts) } -func extractData(pkgReader io.ReadSeeker, openTar OpenTarFunc, options *ExtractOptions) error { - dataReader, err := openTar(pkgReader) +func extractData(pkgReader io.ReadSeeker, opener TarOpener, options *ExtractOptions) error { + dataReader, err := opener(pkgReader) if err != nil { return err } @@ -285,7 +283,7 @@ func extractData(pkgReader io.ReadSeeker, openTar OpenTarFunc, options *ExtractO if err != nil { return err } - err = extractHardLinks(pkgReader, openTar, extractHardLinkOptions) + err = extractHardLinks(pkgReader, opener, extractHardLinkOptions) if err != nil { return err } @@ -319,8 +317,8 @@ type extractHardLinkOptions struct { // extractHardLinks iterates through the tarball a second time to extract the // hard links that were not extracted in the first pass. -func extractHardLinks(pkgReader io.ReadSeeker, openTar OpenTarFunc, opts *extractHardLinkOptions) error { - dataReader, err := openTar(pkgReader) +func extractHardLinks(pkgReader io.ReadSeeker, opener TarOpener, opts *extractHardLinkOptions) error { + dataReader, err := opener(pkgReader) if err != nil { return err } @@ -381,7 +379,7 @@ func extractHardLinks(pkgReader io.ReadSeeker, openTar OpenTarFunc, opts *extrac } // If there are pending links, that means the link targets do not come from - // this tarball. + // this package. if len(opts.pendingLinks) > 0 { var targets []string for target := range opts.pendingLinks { diff --git a/internal/tarball/extract_test.go b/internal/tarball/extract_test.go index 324936095..436bea074 100644 --- a/internal/tarball/extract_test.go +++ b/internal/tarball/extract_test.go @@ -19,7 +19,7 @@ import ( type extractTest struct { summary string pkgdata []byte - openTar tarball.OpenTarFunc + openTar tarball.TarOpener options tarball.ExtractOptions hackopt func(c *C, o *tarball.ExtractOptions) result map[string]string @@ -545,7 +545,7 @@ func (s *S) TestExtract(c *C) { var extractCreateCallbackTests = []struct { summary string pkgdata []byte - openTar tarball.OpenTarFunc + openTar tarball.TarOpener options tarball.ExtractOptions calls map[string][]tarball.ExtractInfo }{{ From ca72fd65d108d3e9d1a10fbadf78228e921b832b Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 8 Sep 2026 15:24:49 +0200 Subject: [PATCH 08/15] refactor: rework approach --- .../cmd_debug_check_release_archives.go | 4 +- internal/archive/archive_test.go | 3 +- internal/deb/extract.go | 49 ---------- internal/slicer/slicer.go | 4 +- internal/tarball/extract.go | 92 +++++++++++++++---- internal/tarball/extract_test.go | 35 ++----- 6 files changed, 88 insertions(+), 99 deletions(-) delete mode 100644 internal/deb/extract.go diff --git a/cmd/chisel/cmd_debug_check_release_archives.go b/cmd/chisel/cmd_debug_check_release_archives.go index 29a8189a8..0225e29f0 100644 --- a/cmd/chisel/cmd_debug_check_release_archives.go +++ b/cmd/chisel/cmd_debug_check_release_archives.go @@ -13,8 +13,8 @@ import ( "github.com/canonical/chisel/internal/archive" "github.com/canonical/chisel/internal/cache" - "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/setup" + "github.com/canonical/chisel/internal/tarball" ) var shortCheckReleaseArchivesHelp = "Check the release's archives" @@ -150,7 +150,7 @@ func computePathObservations(release *setup.Release, archives map[string]archive if err != nil { return nil, err } - dataReader, err := deb.OpenTar(pkgReader) + dataReader, err := tarball.DataReader(pkgReader, tarball.DebFormat) if err != nil { return nil, err } diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index 8c40eeced..bc5687213 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -1236,8 +1236,9 @@ func (s *S) testOpenArchiveArch(c *C, test realArchiveTest, arch string) { c.Assert(info.Name, DeepEquals, test.pkg) c.Assert(info.Arch, DeepEquals, arch) - err = tarball.Extract(pkg, deb.OpenTar, &tarball.ExtractOptions{ + err = tarball.Extract(pkg, &tarball.ExtractOptions{ Package: test.pkg, + Format: tarball.DebFormat, TargetDir: extractDir, Extract: map[string][]tarball.ExtractInfo{ fmt.Sprintf("/usr/share/doc/%s/copyright", test.pkg): { diff --git a/internal/deb/extract.go b/internal/deb/extract.go deleted file mode 100644 index d25f1f883..000000000 --- a/internal/deb/extract.go +++ /dev/null @@ -1,49 +0,0 @@ -package deb - -import ( - "compress/gzip" - "fmt" - "io" - - "github.com/blakesmith/ar" - "github.com/klauspost/compress/zstd" - "github.com/ulikunitz/xz" -) - -// OpenTar takes a Reader for the ar file belonging to a Debian package and -// returns a Reader to the uncompressed inner tarball. -func OpenTar(pkgReader io.Reader) (io.ReadCloser, error) { - arReader := ar.NewReader(pkgReader) - var dataReader io.ReadCloser - for dataReader == nil { - arHeader, err := arReader.Next() - if err == io.EOF { - return nil, fmt.Errorf("no data payload") - } - if err != nil { - return nil, err - } - switch arHeader.Name { - case "data.tar.gz": - gzipReader, err := gzip.NewReader(arReader) - if err != nil { - return nil, err - } - dataReader = gzipReader - case "data.tar.xz": - xzReader, err := xz.NewReader(arReader) - if err != nil { - return nil, err - } - dataReader = io.NopCloser(xzReader) - case "data.tar.zst": - zstdReader, err := zstd.NewReader(arReader) - if err != nil { - return nil, err - } - dataReader = zstdReader.IOReadCloser() - } - } - - return dataReader, nil -} diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 994f65ba2..6c528c651 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -16,7 +16,6 @@ import ( "github.com/klauspost/compress/zstd" "github.com/canonical/chisel/internal/archive" - "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/fsutil" "github.com/canonical/chisel/internal/manifestutil" "github.com/canonical/chisel/internal/scripts" @@ -240,8 +239,9 @@ func Run(options *RunOptions) error { if reader == nil { continue } - err := tarball.Extract(reader, deb.OpenTar, &tarball.ExtractOptions{ + err := tarball.Extract(reader, &tarball.ExtractOptions{ Package: slice.Package, + Format: tarball.DebFormat, Extract: extract[slice.Package], TargetDir: targetDir, Create: create, diff --git a/internal/tarball/extract.go b/internal/tarball/extract.go index 597e8ab96..c8b79c859 100644 --- a/internal/tarball/extract.go +++ b/internal/tarball/extract.go @@ -3,6 +3,7 @@ package tarball import ( "archive/tar" "bytes" + "compress/gzip" "fmt" "io" "io/fs" @@ -12,27 +13,82 @@ import ( "strings" "syscall" + "github.com/blakesmith/ar" + "github.com/klauspost/compress/zstd" "github.com/ulikunitz/xz" "github.com/canonical/chisel/internal/fsutil" "github.com/canonical/chisel/internal/strdist" ) -// TarOpener returns a reader over the uncompressed tar stream contained in -// its input, hiding the container and compression details from Extract. -type TarOpener func(pkgReader io.Reader) (io.ReadCloser, error) +// Format identifies the format of a package. +type Format string -// OpenXZTar returns a reader over the decompressed XZ stream. -func OpenXZTar(pkgReader io.Reader) (io.ReadCloser, error) { - xzReader, err := xz.NewReader(pkgReader) - if err != nil { - return nil, err +const ( + // DebFormat is the Debian package format: an ar archive holding a + // compressed data tarball. + DebFormat Format = "deb" + // BinFormat is the bin package format: a plain XZ-compressed tarball. + BinFormat Format = "bin" +) + +// DataReader returns a reader over the uncompressed tar stream contained in +// the given package, based on its format. +func DataReader(pkgReader io.Reader, format Format) (io.ReadCloser, error) { + switch format { + case DebFormat: + return debDataReader(pkgReader) + case BinFormat: + xzReader, err := xz.NewReader(pkgReader) + if err != nil { + return nil, err + } + return io.NopCloser(xzReader), nil + } + return nil, fmt.Errorf("internal error: unsupported package format: %q", format) +} + +// debDataReader returns a reader over the data tarball contained in the ar +// file of a Debian package. +func debDataReader(pkgReader io.Reader) (io.ReadCloser, error) { + arReader := ar.NewReader(pkgReader) + var dataReader io.ReadCloser + for dataReader == nil { + arHeader, err := arReader.Next() + if err == io.EOF { + return nil, fmt.Errorf("no data payload") + } + if err != nil { + return nil, err + } + switch arHeader.Name { + case "data.tar.gz": + gzipReader, err := gzip.NewReader(arReader) + if err != nil { + return nil, err + } + dataReader = gzipReader + case "data.tar.xz": + xzReader, err := xz.NewReader(arReader) + if err != nil { + return nil, err + } + dataReader = io.NopCloser(xzReader) + case "data.tar.zst": + zstdReader, err := zstd.NewReader(arReader) + if err != nil { + return nil, err + } + dataReader = zstdReader.IOReadCloser() + } } - return io.NopCloser(xzReader), nil + + return dataReader, nil } type ExtractOptions struct { Package string + Format Format TargetDir string Extract map[string][]ExtractInfo // Create can optionally be set to control the creation of extracted entries. @@ -72,7 +128,7 @@ func getValidOptions(options *ExtractOptions) (*ExtractOptions, error) { return options, nil } -func Extract(pkgReader io.ReadSeeker, opener TarOpener, options *ExtractOptions) (err error) { +func Extract(pkgReader io.ReadSeeker, options *ExtractOptions) (err error) { defer func() { if err != nil { err = fmt.Errorf("cannot extract from package %q: %w", options.Package, err) @@ -81,10 +137,6 @@ func Extract(pkgReader io.ReadSeeker, opener TarOpener, options *ExtractOptions) logf("Extracting files from package %q...", options.Package) - if opener == nil { - return fmt.Errorf("internal error: no tar opener provided") - } - validOpts, err := getValidOptions(options) if err != nil { return err @@ -97,11 +149,11 @@ func Extract(pkgReader io.ReadSeeker, opener TarOpener, options *ExtractOptions) return err } - return extractData(pkgReader, opener, validOpts) + return extractData(pkgReader, validOpts) } -func extractData(pkgReader io.ReadSeeker, opener TarOpener, options *ExtractOptions) error { - dataReader, err := opener(pkgReader) +func extractData(pkgReader io.ReadSeeker, options *ExtractOptions) error { + dataReader, err := DataReader(pkgReader, options.Format) if err != nil { return err } @@ -283,7 +335,7 @@ func extractData(pkgReader io.ReadSeeker, opener TarOpener, options *ExtractOpti if err != nil { return err } - err = extractHardLinks(pkgReader, opener, extractHardLinkOptions) + err = extractHardLinks(pkgReader, extractHardLinkOptions) if err != nil { return err } @@ -317,8 +369,8 @@ type extractHardLinkOptions struct { // extractHardLinks iterates through the tarball a second time to extract the // hard links that were not extracted in the first pass. -func extractHardLinks(pkgReader io.ReadSeeker, opener TarOpener, opts *extractHardLinkOptions) error { - dataReader, err := opener(pkgReader) +func extractHardLinks(pkgReader io.ReadSeeker, opts *extractHardLinkOptions) error { + dataReader, err := DataReader(pkgReader, opts.Format) if err != nil { return err } diff --git a/internal/tarball/extract_test.go b/internal/tarball/extract_test.go index 436bea074..0b3b670cb 100644 --- a/internal/tarball/extract_test.go +++ b/internal/tarball/extract_test.go @@ -10,7 +10,6 @@ import ( . "gopkg.in/check.v1" - "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/fsutil" "github.com/canonical/chisel/internal/tarball" "github.com/canonical/chisel/internal/testutil" @@ -19,7 +18,7 @@ import ( type extractTest struct { summary string pkgdata []byte - openTar tarball.TarOpener + format tarball.Format options tarball.ExtractOptions hackopt func(c *C, o *tarball.ExtractOptions) result map[string]string @@ -498,6 +497,9 @@ func (s *S) TestExtract(c *C) { options := test.options options.Package = "test-package" options.TargetDir = dir + if options.Format == "" { + options.Format = tarball.DebFormat + } createdPaths := make(map[string]bool) options.Create = func(_ []tarball.ExtractInfo, o *fsutil.CreateOptions) error { relPath := filepath.Clean("/" + strings.TrimPrefix(o.Path, dir)) @@ -513,11 +515,7 @@ func (s *S) TestExtract(c *C) { test.hackopt(c, &options) } - openTar := test.openTar - if openTar == nil { - openTar = deb.OpenTar - } - err := tarball.Extract(bytes.NewReader(test.pkgdata), openTar, &options) + err := tarball.Extract(bytes.NewReader(test.pkgdata), &options) if test.error != "" { c.Assert(err, ErrorMatches, test.error) continue @@ -545,7 +543,7 @@ func (s *S) TestExtract(c *C) { var extractCreateCallbackTests = []struct { summary string pkgdata []byte - openTar tarball.TarOpener + format tarball.Format options tarball.ExtractOptions calls map[string][]tarball.ExtractInfo }{{ @@ -612,6 +610,9 @@ func (s *S) TestExtractCreateCallback(c *C) { options := test.options options.Package = "test-package" options.TargetDir = dir + if options.Format == "" { + options.Format = tarball.DebFormat + } createExtractInfos := map[string][]tarball.ExtractInfo{} options.Create = func(extractInfos []tarball.ExtractInfo, o *fsutil.CreateOptions) error { if extractInfos == nil { @@ -629,25 +630,9 @@ func (s *S) TestExtractCreateCallback(c *C) { return nil } - openTar := test.openTar - if openTar == nil { - openTar = deb.OpenTar - } - err := tarball.Extract(bytes.NewReader(test.pkgdata), openTar, &options) + err := tarball.Extract(bytes.NewReader(test.pkgdata), &options) c.Assert(err, IsNil) c.Assert(createExtractInfos, DeepEquals, test.calls) } } - -func (s *S) TestExtractMissingOpenTar(c *C) { - options := tarball.ExtractOptions{ - Package: "test-package", - TargetDir: c.MkDir(), - Extract: map[string][]tarball.ExtractInfo{ - "/dir/file": {{Path: "/dir/file"}}, - }, - } - err := tarball.Extract(bytes.NewReader(testutil.PackageData["test-package"]), nil, &options) - c.Assert(err, ErrorMatches, `cannot extract from package "test-package": internal error: no tar opener provided`) -} From dc8e94bd6caedca91cb578482c7fbeef33c6266c Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 8 Sep 2026 15:40:05 +0200 Subject: [PATCH 09/15] fix: clean superfluous import --- internal/archive/archive_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index bc5687213..3a83d73d5 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -20,7 +20,6 @@ import ( "github.com/canonical/chisel/internal/archive" "github.com/canonical/chisel/internal/archive/testarchive" - "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/tarball" "github.com/canonical/chisel/internal/testutil" ) From f882d4711b3bf0c15f56f9ed419a4409deccbf7a Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 8 Sep 2026 15:50:27 +0200 Subject: [PATCH 10/15] test: remove unused field for now --- internal/tarball/extract_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/tarball/extract_test.go b/internal/tarball/extract_test.go index 0b3b670cb..5f233a92a 100644 --- a/internal/tarball/extract_test.go +++ b/internal/tarball/extract_test.go @@ -18,7 +18,6 @@ import ( type extractTest struct { summary string pkgdata []byte - format tarball.Format options tarball.ExtractOptions hackopt func(c *C, o *tarball.ExtractOptions) result map[string]string From 116752e32f3f83aac412e8a512be3dfe1ea74063 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 9 Sep 2026 08:32:37 +0200 Subject: [PATCH 11/15] refactor: put deb DataReader back in deb package --- internal/deb/extract.go | 49 +++++++++++++++++++++++++++++++++++++ internal/tarball/extract.go | 44 ++------------------------------- 2 files changed, 51 insertions(+), 42 deletions(-) create mode 100644 internal/deb/extract.go diff --git a/internal/deb/extract.go b/internal/deb/extract.go new file mode 100644 index 000000000..2bce509f4 --- /dev/null +++ b/internal/deb/extract.go @@ -0,0 +1,49 @@ +package deb + +import ( + "compress/gzip" + "fmt" + "io" + + "github.com/blakesmith/ar" + "github.com/klauspost/compress/zstd" + "github.com/ulikunitz/xz" +) + +// DataReader takes a Reader for the ar file belonging to a Debian package and +// returns a Reader to the inner tarball. +func DataReader(pkgReader io.Reader) (io.ReadCloser, error) { + arReader := ar.NewReader(pkgReader) + var dataReader io.ReadCloser + for dataReader == nil { + arHeader, err := arReader.Next() + if err == io.EOF { + return nil, fmt.Errorf("no data payload") + } + if err != nil { + return nil, err + } + switch arHeader.Name { + case "data.tar.gz": + gzipReader, err := gzip.NewReader(arReader) + if err != nil { + return nil, err + } + dataReader = gzipReader + case "data.tar.xz": + xzReader, err := xz.NewReader(arReader) + if err != nil { + return nil, err + } + dataReader = io.NopCloser(xzReader) + case "data.tar.zst": + zstdReader, err := zstd.NewReader(arReader) + if err != nil { + return nil, err + } + dataReader = zstdReader.IOReadCloser() + } + } + + return dataReader, nil +} \ No newline at end of file diff --git a/internal/tarball/extract.go b/internal/tarball/extract.go index c8b79c859..3807d1544 100644 --- a/internal/tarball/extract.go +++ b/internal/tarball/extract.go @@ -3,7 +3,6 @@ package tarball import ( "archive/tar" "bytes" - "compress/gzip" "fmt" "io" "io/fs" @@ -13,10 +12,9 @@ import ( "strings" "syscall" - "github.com/blakesmith/ar" - "github.com/klauspost/compress/zstd" "github.com/ulikunitz/xz" + "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/fsutil" "github.com/canonical/chisel/internal/strdist" ) @@ -37,7 +35,7 @@ const ( func DataReader(pkgReader io.Reader, format Format) (io.ReadCloser, error) { switch format { case DebFormat: - return debDataReader(pkgReader) + return deb.DataReader(pkgReader) case BinFormat: xzReader, err := xz.NewReader(pkgReader) if err != nil { @@ -48,44 +46,6 @@ func DataReader(pkgReader io.Reader, format Format) (io.ReadCloser, error) { return nil, fmt.Errorf("internal error: unsupported package format: %q", format) } -// debDataReader returns a reader over the data tarball contained in the ar -// file of a Debian package. -func debDataReader(pkgReader io.Reader) (io.ReadCloser, error) { - arReader := ar.NewReader(pkgReader) - var dataReader io.ReadCloser - for dataReader == nil { - arHeader, err := arReader.Next() - if err == io.EOF { - return nil, fmt.Errorf("no data payload") - } - if err != nil { - return nil, err - } - switch arHeader.Name { - case "data.tar.gz": - gzipReader, err := gzip.NewReader(arReader) - if err != nil { - return nil, err - } - dataReader = gzipReader - case "data.tar.xz": - xzReader, err := xz.NewReader(arReader) - if err != nil { - return nil, err - } - dataReader = io.NopCloser(xzReader) - case "data.tar.zst": - zstdReader, err := zstd.NewReader(arReader) - if err != nil { - return nil, err - } - dataReader = zstdReader.IOReadCloser() - } - } - - return dataReader, nil -} - type ExtractOptions struct { Package string Format Format From e9c4d881ab915066a7a8807bb78a506418d44341 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 9 Sep 2026 08:34:03 +0200 Subject: [PATCH 12/15] style: add missing trailing newline --- internal/deb/extract.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/deb/extract.go b/internal/deb/extract.go index 2bce509f4..88fc4ad56 100644 --- a/internal/deb/extract.go +++ b/internal/deb/extract.go @@ -46,4 +46,4 @@ func DataReader(pkgReader io.Reader) (io.ReadCloser, error) { } return dataReader, nil -} \ No newline at end of file +} From b1ec3903276a6de50ddeec6703775323858ca56d Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 9 Sep 2026 13:40:51 +0200 Subject: [PATCH 13/15] style: refine naming --- cmd/chisel/cmd_debug_check_release_archives.go | 2 +- internal/tarball/extract.go | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/chisel/cmd_debug_check_release_archives.go b/cmd/chisel/cmd_debug_check_release_archives.go index 0225e29f0..c7931d9a6 100644 --- a/cmd/chisel/cmd_debug_check_release_archives.go +++ b/cmd/chisel/cmd_debug_check_release_archives.go @@ -150,7 +150,7 @@ func computePathObservations(release *setup.Release, archives map[string]archive if err != nil { return nil, err } - dataReader, err := tarball.DataReader(pkgReader, tarball.DebFormat) + dataReader, err := tarball.DebFormat.TarStream(pkgReader) if err != nil { return nil, err } diff --git a/internal/tarball/extract.go b/internal/tarball/extract.go index 3807d1544..0760a1fef 100644 --- a/internal/tarball/extract.go +++ b/internal/tarball/extract.go @@ -30,9 +30,9 @@ const ( BinFormat Format = "bin" ) -// DataReader returns a reader over the uncompressed tar stream contained in -// the given package, based on its format. -func DataReader(pkgReader io.Reader, format Format) (io.ReadCloser, error) { +// TarStream returns a reader over the uncompressed tar stream contained +// in the given package. +func (format Format) TarStream(pkgReader io.Reader) (io.ReadCloser, error) { switch format { case DebFormat: return deb.DataReader(pkgReader) @@ -113,7 +113,7 @@ func Extract(pkgReader io.ReadSeeker, options *ExtractOptions) (err error) { } func extractData(pkgReader io.ReadSeeker, options *ExtractOptions) error { - dataReader, err := DataReader(pkgReader, options.Format) + dataReader, err := options.Format.TarStream(pkgReader) if err != nil { return err } @@ -330,7 +330,7 @@ type extractHardLinkOptions struct { // extractHardLinks iterates through the tarball a second time to extract the // hard links that were not extracted in the first pass. func extractHardLinks(pkgReader io.ReadSeeker, opts *extractHardLinkOptions) error { - dataReader, err := DataReader(pkgReader, opts.Format) + dataReader, err := opts.Format.TarStream(pkgReader) if err != nil { return err } From a8ad8137611ca866f6f3d33f2be2d44fa6ad3082 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 9 Sep 2026 13:46:18 +0200 Subject: [PATCH 14/15] style: improve naming consistency --- internal/tarball/extract.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/tarball/extract.go b/internal/tarball/extract.go index 0760a1fef..f61ed21e1 100644 --- a/internal/tarball/extract.go +++ b/internal/tarball/extract.go @@ -113,11 +113,11 @@ func Extract(pkgReader io.ReadSeeker, options *ExtractOptions) (err error) { } func extractData(pkgReader io.ReadSeeker, options *ExtractOptions) error { - dataReader, err := options.Format.TarStream(pkgReader) + tarStream, err := options.Format.TarStream(pkgReader) if err != nil { return err } - defer dataReader.Close() + defer tarStream.Close() oldUmask := syscall.Umask(0) defer func() { @@ -150,7 +150,7 @@ func extractData(pkgReader io.ReadSeeker, options *ExtractOptions) error { // before the entry for the file itself. This is the case for the tarballs // produced by common packaging tools but not for all tarballs. tarDirMode := make(map[string]fs.FileMode) - tarReader := tar.NewReader(dataReader) + tarReader := tar.NewReader(tarStream) for { tarHeader, err := tarReader.Next() if err == io.EOF { @@ -330,13 +330,13 @@ type extractHardLinkOptions struct { // extractHardLinks iterates through the tarball a second time to extract the // hard links that were not extracted in the first pass. func extractHardLinks(pkgReader io.ReadSeeker, opts *extractHardLinkOptions) error { - dataReader, err := opts.Format.TarStream(pkgReader) + tarStream, err := opts.Format.TarStream(pkgReader) if err != nil { return err } - defer dataReader.Close() + defer tarStream.Close() - tarReader := tar.NewReader(dataReader) + tarReader := tar.NewReader(tarStream) for { tarHeader, err := tarReader.Next() if err == io.EOF { From 6ad029e50f078420f157ddcfa850e11f070c53d7 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 9 Sep 2026 13:54:03 +0200 Subject: [PATCH 15/15] style: avoid ambiguity --- internal/tarball/extract.go | 16 ++++++++-------- internal/tarball/extract_test.go | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/internal/tarball/extract.go b/internal/tarball/extract.go index f61ed21e1..a0c7af11a 100644 --- a/internal/tarball/extract.go +++ b/internal/tarball/extract.go @@ -19,20 +19,20 @@ import ( "github.com/canonical/chisel/internal/strdist" ) -// Format identifies the format of a package. -type Format string +// PkgFormat identifies the format of a package. +type PkgFormat string const ( // DebFormat is the Debian package format: an ar archive holding a // compressed data tarball. - DebFormat Format = "deb" + DebFormat PkgFormat = "deb" // BinFormat is the bin package format: a plain XZ-compressed tarball. - BinFormat Format = "bin" + BinFormat PkgFormat = "bin" ) -// TarStream returns a reader over the uncompressed tar stream contained -// in the given package. -func (format Format) TarStream(pkgReader io.Reader) (io.ReadCloser, error) { +// TarStream returns a reader over the raw, uncompressed tar stream contained +// in the given package. The stream is not parsed. +func (format PkgFormat) TarStream(pkgReader io.Reader) (io.ReadCloser, error) { switch format { case DebFormat: return deb.DataReader(pkgReader) @@ -48,7 +48,7 @@ func (format Format) TarStream(pkgReader io.Reader) (io.ReadCloser, error) { type ExtractOptions struct { Package string - Format Format + Format PkgFormat TargetDir string Extract map[string][]ExtractInfo // Create can optionally be set to control the creation of extracted entries. diff --git a/internal/tarball/extract_test.go b/internal/tarball/extract_test.go index 5f233a92a..ed43414ef 100644 --- a/internal/tarball/extract_test.go +++ b/internal/tarball/extract_test.go @@ -542,7 +542,7 @@ func (s *S) TestExtract(c *C) { var extractCreateCallbackTests = []struct { summary string pkgdata []byte - format tarball.Format + format tarball.PkgFormat options tarball.ExtractOptions calls map[string][]tarball.ExtractInfo }{{