diff --git a/get/syncer.go b/get/syncer.go index 292cfa4..5ae3f68 100644 --- a/get/syncer.go +++ b/get/syncer.go @@ -12,15 +12,17 @@ import ( "log" "net/url" "path" + "path/filepath" "strings" + "github.com/klauspost/compress/zstd" "github.com/uyuni-project/minima/util" "golang.org/x/crypto/openpgp" ) // common -// XMLLocation maps a tag in repodata/repomd.xml or repodata/-primary.xml.gz +// XMLLocation maps a tag in repodata/repomd.xml or repodata/-primary.xml. type XMLLocation struct { Href string `xml:"href,attr"` } @@ -39,21 +41,21 @@ type XMLData struct { Checksum XMLChecksum `xml:"checksum"` } -// repodata/-primary.xml.gz +// repodata/-primary.xml. -// XMLMetaData maps a tag in repodata/-primary.xml.gz +// XMLMetaData maps a tag in repodata/-primary.xml. type XMLMetaData struct { Packages []XMLPackage `xml:"package"` } -// XMLPackage maps a tag in repodata/-primary.xml.gz +// XMLPackage maps a tag in repodata/-primary.xml. type XMLPackage struct { Arch string `xml:"arch"` Location XMLLocation `xml:"location"` Checksum XMLChecksum `xml:"checksum"` } -// XMLChecksum maps a tag in repodata/-primary.xml.gz +// XMLChecksum maps a tag in repodata/-primary.xml. type XMLChecksum struct { Type string `xml:"type,attr"` Checksum string `xml:",cdata"` @@ -72,7 +74,7 @@ type RepoType struct { MetadataPath string PackagesType string DecodeMetadata func(io.Reader) (XMLRepomd, error) - DecodePackages func(io.Reader) (XMLMetaData, error) + DecodePackages func(io.Reader, string) (XMLMetaData, error) MetadataSignatureExt string Noarch string } @@ -313,19 +315,35 @@ func (e *SignatureError) Error() string { return fmt.Sprintf("Signature error: %s", e.reason) } -func readMetaData(reader io.Reader) (primary XMLMetaData, err error) { - gzReader, err := gzip.NewReader(reader) - if err != nil { - return - } - defer gzReader.Close() +// Uncompress and read primary XML +func readMetaData(reader io.Reader, compType string) (XMLMetaData, error) { + var primary XMLMetaData + switch compType { + case "gz": + reader, err := gzip.NewReader(reader) + if err != nil { + return primary, err + } + defer reader.Close() - decoder := xml.NewDecoder(gzReader) - err = decoder.Decode(&primary) + decoder := xml.NewDecoder(reader) + err = decoder.Decode(&primary) + case "zst": + reader, err := zstd.NewReader(reader) + if err != nil { + return primary, err + } + defer reader.Close() - return + decoder := xml.NewDecoder(reader) + err = decoder.Decode(&primary) + default: + return primary, errors.New("Unsupported compression type") + } + return primary, nil } + func (r *Syncer) readChecksumMap() (checksumMap map[string]XMLChecksum) { checksumMap = make(map[string]XMLChecksum) repomdReader, err := r.storage.NewReader(repomdPath, Permanent) @@ -361,7 +379,8 @@ func (r *Syncer) readChecksumMap() (checksumMap map[string]XMLChecksum) { if err != nil { return } - primary, err := repoType.DecodePackages(primaryReader) + compType := strings.Trim(filepath.Ext(dataHref), ".") + primary, err := repoType.DecodePackages(primaryReader, compType) if err != nil { return } @@ -380,7 +399,8 @@ func (r *Syncer) processPrimary(path string, checksumMap map[string]XMLChecksum, if err != nil { return } - primary, err := repoType.DecodePackages(reader) + compType := strings.Trim(filepath.Ext(path), ".") + primary, err := repoType.DecodePackages(reader, compType) if err != nil { return } @@ -473,7 +493,7 @@ func decodeRelease(reader io.Reader) (repomd XMLRepomd, err error) { return } -func decodePackages(reader io.Reader) (metadata XMLMetaData, err error) { +func decodePackages(reader io.Reader, _ string) (metadata XMLMetaData, err error) { packagesEntries, err := util.ProcessPropertiesFile(reader) if err != nil { return diff --git a/get/syncer_test.go b/get/syncer_test.go index 8f6f805..718f6c8 100644 --- a/get/syncer_test.go +++ b/get/syncer_test.go @@ -68,6 +68,62 @@ func TestStoreRepo(t *testing.T) { } } +func TestStoreRepoZstd(t *testing.T) { + directory := filepath.Join(os.TempDir(), "syncer_test") + err := os.RemoveAll(directory) + if err != nil { + t.Error(err) + } + + archs := map[string]bool{ + "x86_64": true, + } + storage := NewFileStorage(directory) + url, err := url.Parse("http://localhost:8080/zstrepo") + if err != nil { + t.Error(err) + } + syncer := NewSyncer(*url, archs, storage) + + // first sync + err = syncer.StoreRepo() + if err != nil { + t.Error(err) + } + + expectedFiles := []string{ + filepath.Join("repodata", "106c411e443c6b97e7d547f78e32d6d2cdf8e80577999954fdb770d8a21581a0-filelists.xml.zst"), + filepath.Join("repodata", "d7fd4cf502d9e1ab8865bc9690c34c3cadcfda0db269f4ddc9c6f026db3f2400-primary.xml.zst"), + filepath.Join("repodata", "fa0767ea9359279bb6e8ec93a70cefb3863e4fa4bcbeebbfe755a5ce16c21b94-other.xml.zst"), + filepath.Join("repodata", "repomd.xml"), + filepath.Join("x86_64", "milkyway-dummy-2.0-1.1.x86_64.rpm"), + filepath.Join("x86_64", "orion-dummy-1.1-1.1.x86_64.rpm"), + filepath.Join("x86_64", "hoag-dummy-1.1-2.1.x86_64.rpm"), + filepath.Join("x86_64", "perseus-dummy-1.1-1.1.x86_64.rpm"), + filepath.Join("x86_64", "orion-dummy-sle12-1.1-4.1.x86_64.rpm"), + } + + for _, file := range expectedFiles { + originalInfo, serr := os.Stat(filepath.Join("testdata", "zstrepo", file)) + if err != nil { + t.Fatal(serr) + } + syncedInfo, serr := os.Stat(filepath.Join(directory, file)) + if serr != nil { + t.Fatal(serr) + } + if originalInfo.Size() != syncedInfo.Size() { + t.Error("original and synced versions of", file, "differ:", originalInfo.Size(), "vs", syncedInfo.Size()) + } + } + + // second sync + err = syncer.StoreRepo() + if err != nil { + t.Error(err) + } +} + func TestStoreDebRepo(t *testing.T) { directory := filepath.Join(os.TempDir(), "syncer_test") err := os.RemoveAll(directory) diff --git a/get/testdata/zstrepo/repodata/106c411e443c6b97e7d547f78e32d6d2cdf8e80577999954fdb770d8a21581a0-filelists.xml.zst b/get/testdata/zstrepo/repodata/106c411e443c6b97e7d547f78e32d6d2cdf8e80577999954fdb770d8a21581a0-filelists.xml.zst new file mode 100644 index 0000000..d55a678 Binary files /dev/null and b/get/testdata/zstrepo/repodata/106c411e443c6b97e7d547f78e32d6d2cdf8e80577999954fdb770d8a21581a0-filelists.xml.zst differ diff --git a/get/testdata/zstrepo/repodata/d7fd4cf502d9e1ab8865bc9690c34c3cadcfda0db269f4ddc9c6f026db3f2400-primary.xml.zst b/get/testdata/zstrepo/repodata/d7fd4cf502d9e1ab8865bc9690c34c3cadcfda0db269f4ddc9c6f026db3f2400-primary.xml.zst new file mode 100644 index 0000000..0b4d81f Binary files /dev/null and b/get/testdata/zstrepo/repodata/d7fd4cf502d9e1ab8865bc9690c34c3cadcfda0db269f4ddc9c6f026db3f2400-primary.xml.zst differ diff --git a/get/testdata/zstrepo/repodata/fa0767ea9359279bb6e8ec93a70cefb3863e4fa4bcbeebbfe755a5ce16c21b94-other.xml.zst b/get/testdata/zstrepo/repodata/fa0767ea9359279bb6e8ec93a70cefb3863e4fa4bcbeebbfe755a5ce16c21b94-other.xml.zst new file mode 100644 index 0000000..b8bf625 Binary files /dev/null and b/get/testdata/zstrepo/repodata/fa0767ea9359279bb6e8ec93a70cefb3863e4fa4bcbeebbfe755a5ce16c21b94-other.xml.zst differ diff --git a/get/testdata/zstrepo/repodata/repomd.xml b/get/testdata/zstrepo/repodata/repomd.xml new file mode 100644 index 0000000..bd9fa75 --- /dev/null +++ b/get/testdata/zstrepo/repodata/repomd.xml @@ -0,0 +1,28 @@ + + + 1720081452 + + d7fd4cf502d9e1ab8865bc9690c34c3cadcfda0db269f4ddc9c6f026db3f2400 + cff719a5cea867237259b112623d7d7e5804c863dd41dadfe52256593aaee640 + + 1720081452 + 1347 + 7245 + + + 106c411e443c6b97e7d547f78e32d6d2cdf8e80577999954fdb770d8a21581a0 + dd55851e15239a7d86bf045e7729259b6c0dd884e202a651eac05c4dd014fd40 + + 1720081452 + 566 + 1715 + + + fa0767ea9359279bb6e8ec93a70cefb3863e4fa4bcbeebbfe755a5ce16c21b94 + 9285a7e58c6870843af0f7cf069592be7ea7dd087d4836c0485f5d170e997ce5 + + 1720081452 + 573 + 1660 + + diff --git a/get/testdata/zstrepo/x86_64/hoag-dummy-1.1-2.1.x86_64.rpm b/get/testdata/zstrepo/x86_64/hoag-dummy-1.1-2.1.x86_64.rpm new file mode 100644 index 0000000..b1200db Binary files /dev/null and b/get/testdata/zstrepo/x86_64/hoag-dummy-1.1-2.1.x86_64.rpm differ diff --git a/get/testdata/zstrepo/x86_64/milkyway-dummy-2.0-1.1.x86_64.rpm b/get/testdata/zstrepo/x86_64/milkyway-dummy-2.0-1.1.x86_64.rpm new file mode 100644 index 0000000..7567824 Binary files /dev/null and b/get/testdata/zstrepo/x86_64/milkyway-dummy-2.0-1.1.x86_64.rpm differ diff --git a/get/testdata/zstrepo/x86_64/orion-dummy-1.1-1.1.x86_64.rpm b/get/testdata/zstrepo/x86_64/orion-dummy-1.1-1.1.x86_64.rpm new file mode 100644 index 0000000..a717b3e Binary files /dev/null and b/get/testdata/zstrepo/x86_64/orion-dummy-1.1-1.1.x86_64.rpm differ diff --git a/get/testdata/zstrepo/x86_64/orion-dummy-sle12-1.1-4.1.x86_64.rpm b/get/testdata/zstrepo/x86_64/orion-dummy-sle12-1.1-4.1.x86_64.rpm new file mode 100644 index 0000000..26c5208 Binary files /dev/null and b/get/testdata/zstrepo/x86_64/orion-dummy-sle12-1.1-4.1.x86_64.rpm differ diff --git a/get/testdata/zstrepo/x86_64/perseus-dummy-1.1-1.1.x86_64.rpm b/get/testdata/zstrepo/x86_64/perseus-dummy-1.1-1.1.x86_64.rpm new file mode 100644 index 0000000..4a6ee06 Binary files /dev/null and b/get/testdata/zstrepo/x86_64/perseus-dummy-1.1-1.1.x86_64.rpm differ diff --git a/go.mod b/go.mod index 13d7a76..09fe969 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7 // indirect github.com/jtolds/gls v4.2.1+incompatible // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/kr/pretty v0.1.0 // indirect github.com/smartystreets/assertions v0.0.0-20170818220048-9c0ea8acbc1d // indirect github.com/smartystreets/goconvey v0.0.0-20170825221426-e5b2b7c91115 // indirect diff --git a/go.sum b/go.sum index 7e958ee..908b84a 100644 --- a/go.sum +++ b/go.sum @@ -13,6 +13,8 @@ github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7 h1:SMvOWPJCES github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=