Skip to content

Commit 6ef788a

Browse files
madonukomadonuko
committed
feat(strm): RegPkg to add pkgs to existing streams
1 parent a06f1a4 commit 6ef788a

File tree

4 files changed

+165
-97
lines changed

4 files changed

+165
-97
lines changed

‎db/db.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ type Stream struct {
3434
Fetch string `json:"fetch"`
3535
Forge ForgeType `json:"forge"`
3636
Ver string `json:"ver"`
37-
Mirrors string `json:"mirrors"`
37+
Mirrors string `json:"mirrors"` // comma-separated list of mirrors
3838
}
3939

4040
// Pkg is the package model. Uses UUID primary key instead of gorm.Model's uint.
@@ -55,8 +55,8 @@ type Pkg struct {
5555

5656
Meta datatypes.JSON `gorm:"type:jsonb;default:'{}'" json:"meta"`
5757

58-
StreamID string `json:"stream_id"`
59-
Stream Stream `json:"-"`
58+
StreamID *uuid.UUID `json:"stream_id"`
59+
Stream Stream `json:"-"`
6060
}
6161

6262
// Repo represents a package repository. Its ID is a string.

‎db/strm.go‎

Lines changed: 68 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,77 @@
11
// Determine stream of a package.
22
package db
33

4-
func UpTrace(p Pkg) (urls []string) {
4+
import (
5+
"encoding/json"
6+
"errors"
7+
"maps"
8+
"slices"
9+
"strings"
10+
11+
"github.com/mdobak/go-xerrors"
12+
"github.com/terrapkg/gura/repomd"
13+
"github.com/terrapkg/gura/util"
14+
"go.uber.org/zap"
15+
"gorm.io/gorm"
16+
)
17+
18+
func RegPkg(p Pkg) error {
19+
if p.StreamID != nil {
20+
return DB.Save(p).Error
21+
}
22+
url_ch := make(chan string)
23+
UpTrace(p, url_ch)
24+
close(url_ch)
25+
urls := map[string]struct{}{}
26+
var strm *Stream
27+
recv:
28+
url, ok := <-url_ch
29+
if !ok {
30+
// TODO: logic for handling new stream
31+
goto done
32+
}
33+
if e := DB.Find(&strm, "? IN STRING_TO_ARRAY(mirrors)", url).Error; e != nil {
34+
if errors.Is(e, gorm.ErrRecordNotFound) {
35+
goto recv
36+
}
37+
return xerrors.Newf("error finding stream in db (pkgid %s): %w", p.ID.String(), e)
38+
}
39+
done:
40+
for url := range strings.SplitSeq(strm.Mirrors, ",") {
41+
urls[url] = struct{}{}
42+
}
43+
for url, ok := <-url_ch; ok; {
44+
urls[url] = struct{}{}
45+
}
46+
47+
strm.Mirrors = strings.Join(slices.Collect(maps.Keys(urls)), ",")
48+
p.StreamID = &strm.ID
49+
if err := DB.Save(p).Error; err != nil {
50+
return err
51+
}
52+
if err := DB.Save(strm).Error; err != nil {
53+
return err
54+
}
55+
return nil
56+
}
57+
58+
func UpTrace(p Pkg, url_ch chan string) {
559
switch p.Repo.Type {
660
case Rpm:
7-
return rpmTrace(p)
61+
rpmTrace(p, url_ch)
62+
default:
63+
l.DPanic("unreachable in uptrace")
864
}
9-
l.DPanic("unreachable in uptrace")
10-
return
1165
}
1266

13-
func rpmTrace(p Pkg) (urls []string) {
14-
// p.Meta
15-
return
67+
func rpmTrace(p Pkg, url_ch chan string) {
68+
bs, err := p.Meta.MarshalJSON()
69+
if util.Yeet(l, "error marshaling package metadata", err, zap.String("id", p.ID.String())) {
70+
l.DPanic("DPanic")
71+
return
72+
}
73+
74+
var meta repomd.RPMMeta
75+
json.Unmarshal(bs, &meta)
76+
url_ch <- meta.Url
1677
}

‎kudari/rpm.go‎

Lines changed: 3 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -18,89 +18,10 @@ import (
1818
"github.com/klauspost/compress/zstd"
1919
"github.com/terrapkg/gura/db"
2020
"github.com/terrapkg/gura/util"
21+
. "github.com/terrapkg/gura/repomd"
2122
"go.uber.org/zap"
2223
)
2324

24-
// Represent the root structure of repomd.xml metadata for an RPM repository
25-
// It contains information about available metadata files and their checksums.
26-
type RPMRepomd struct {
27-
XMLName xml.Name `xml:"repomd"`
28-
Revision string `xml:"revision"`
29-
Data []struct {
30-
Type string `xml:"type,attr"`
31-
Checksum RPMChecksum `xml:"checksum"`
32-
OpenChecksum *RPMChecksum `xml:"open-checksum,omitempty"`
33-
HeaderChecksum *RPMChecksum `xml:"header-checksum,omitempty"`
34-
Timestamp int64 `xml:"timestamp"`
35-
Size int64 `xml:"size"`
36-
OpenSize *int64 `xml:"open-size,omitempty"`
37-
HeaderSize *int64 `xml:"header-size,omitempty"`
38-
DatabaseVersion *int `xml:"database_version,omitempty"`
39-
Location struct {
40-
Href string `xml:"href,attr"`
41-
} `xml:"location"`
42-
} `xml:"data"`
43-
}
44-
45-
// Checksum value and its type for repository metadata
46-
type RPMChecksum struct {
47-
Type string `xml:"type,attr"`
48-
Value string `xml:"chardata"`
49-
}
50-
51-
// Single package entry in primary.xml metadata
52-
//
53-
// Includes basic package information, versioning, checksums, and additional metadata in the Format field.
54-
type RPMPackageXML struct {
55-
Name string `xml:"name"`
56-
Arch string `xml:"arch"`
57-
Version struct {
58-
Epoch string `xml:"epoch,attr"`
59-
Ver string `xml:"ver,attr"`
60-
Rel string `xml:"rel,attr"`
61-
} `xml:"version"`
62-
Checksum RPMChecksum `xml:"checksum"`
63-
Packager string `xml:"packager"`
64-
Url string `xml:"url"`
65-
// time
66-
// size
67-
// location
68-
Format RPMFormat `xml:"format"`
69-
}
70-
71-
// Additional metadata for an RPM package
72-
type RPMFormat struct {
73-
License *string `xml:"license,omitempty"`
74-
Vendor *string `xml:"vendor,omitempty"`
75-
Group *string `xml:"group,omitempty"`
76-
Buildhost *string `xml:"buildhost,omitempty"`
77-
Sourcerpm *string `xml:"sourcerpm,omitempty"`
78-
Provides []RPMEntry `xml:"provides>entry,omitempty"`
79-
Requires []RPMEntry `xml:"requires>entry,omitempty"`
80-
Obsoletes []RPMEntry `xml:"obsoletes>entry,omitempty"`
81-
Conflicts []RPMEntry `xml:"conflicts>entry,omitempty"`
82-
Enhances []RPMEntry `xml:"enhances>entry,omitempty"`
83-
Suggests []RPMEntry `xml:"suggests>entry,omitempty"`
84-
Recommends []RPMEntry `xml:"recommends>entry,omitempty"`
85-
Supplements []RPMEntry `xml:"supplements>entry,omitempty"`
86-
}
87-
88-
// Single dependency or capability entry in RPM metadata
89-
type RPMEntry struct {
90-
Name string `xml:"name,attr"`
91-
Flags string `xml:"flags,attr"`
92-
Epoch string `xml:"epoch,attr"`
93-
Ver string `xml:"ver,attr"`
94-
Rel string `xml:"rel,attr"`
95-
}
96-
97-
// Root structure of primary.xml metadata
98-
//
99-
// Contains a list of all packages available in the repository.
100-
type RPMPrimaryXML struct {
101-
Packages []RPMPackageXML `xml:"package"`
102-
}
103-
10425
// Fetch and decode the repomd.xml file from the specified repository URL
10526
//
10627
// On failure, return nil
@@ -266,7 +187,7 @@ func rpmFetch(repo db.Repo) {
266187
}
267188
pkgs[n].FullVer = fullver
268189
pkgs[n].Ver = p.Version.Ver
269-
pkgs[n].Meta = rpm2MetaJSON(p)
190+
util.MaybeSuicide(l, "Meta.UnmarshalJSON", pkgs[n].Meta.UnmarshalJSON(rpm2MetaJSON(p)))
270191
tx.Save(&pkgs[n])
271192
updated++
272193
} else {
@@ -315,12 +236,7 @@ func rpmCompare(a, b RPMPackageXML) int {
315236

316237
// Serialize all fields of [RPMPackageXML] except Name, Arch, and Version into JSON for storage in [db.Pkg.Meta].
317238
func rpm2MetaJSON(p RPMPackageXML) []byte {
318-
bs, err := json.Marshal(struct {
319-
Checksum RPMChecksum
320-
Packager string
321-
Url string
322-
Format RPMFormat
323-
}{
239+
bs, err := json.Marshal(RPMMeta{
324240
Checksum: p.Checksum,
325241
Packager: p.Packager,
326242
Url: p.Url,

‎repomd/rpm.go‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
package repomd
2+
3+
import "encoding/xml"
4+
5+
// Represent the root structure of repomd.xml metadata for an RPM repository
6+
// It contains information about available metadata files and their checksums.
7+
type RPMRepomd struct {
8+
XMLName xml.Name `xml:"repomd"`
9+
Revision string `xml:"revision"`
10+
Data []struct {
11+
Type string `xml:"type,attr"`
12+
Checksum RPMChecksum `xml:"checksum"`
13+
OpenChecksum *RPMChecksum `xml:"open-checksum,omitempty"`
14+
HeaderChecksum *RPMChecksum `xml:"header-checksum,omitempty"`
15+
Timestamp int64 `xml:"timestamp"`
16+
Size int64 `xml:"size"`
17+
OpenSize *int64 `xml:"open-size,omitempty"`
18+
HeaderSize *int64 `xml:"header-size,omitempty"`
19+
DatabaseVersion *int `xml:"database_version,omitempty"`
20+
Location struct {
21+
Href string `xml:"href,attr"`
22+
} `xml:"location"`
23+
} `xml:"data"`
24+
}
25+
26+
// Checksum value and its type for repository metadata
27+
type RPMChecksum struct {
28+
Type string `xml:"type,attr"`
29+
Value string `xml:"chardata"`
30+
}
31+
32+
// Single package entry in primary.xml metadata
33+
//
34+
// Includes basic package information, versioning, checksums, and additional metadata in the Format field.
35+
type RPMPackageXML struct {
36+
Name string `xml:"name"`
37+
Arch string `xml:"arch"`
38+
Version struct {
39+
Epoch string `xml:"epoch,attr"`
40+
Ver string `xml:"ver,attr"`
41+
Rel string `xml:"rel,attr"`
42+
} `xml:"version"`
43+
Checksum RPMChecksum `xml:"checksum"`
44+
Packager string `xml:"packager"`
45+
Url string `xml:"url"`
46+
// time
47+
// size
48+
// location
49+
Format RPMFormat `xml:"format"`
50+
}
51+
52+
// Additional metadata for an RPM package
53+
type RPMFormat struct {
54+
License *string `xml:"license,omitempty"`
55+
Vendor *string `xml:"vendor,omitempty"`
56+
Group *string `xml:"group,omitempty"`
57+
Buildhost *string `xml:"buildhost,omitempty"`
58+
Sourcerpm *string `xml:"sourcerpm,omitempty"`
59+
Provides []RPMEntry `xml:"provides>entry,omitempty"`
60+
Requires []RPMEntry `xml:"requires>entry,omitempty"`
61+
Obsoletes []RPMEntry `xml:"obsoletes>entry,omitempty"`
62+
Conflicts []RPMEntry `xml:"conflicts>entry,omitempty"`
63+
Enhances []RPMEntry `xml:"enhances>entry,omitempty"`
64+
Suggests []RPMEntry `xml:"suggests>entry,omitempty"`
65+
Recommends []RPMEntry `xml:"recommends>entry,omitempty"`
66+
Supplements []RPMEntry `xml:"supplements>entry,omitempty"`
67+
}
68+
69+
// Single dependency or capability entry in RPM metadata
70+
type RPMEntry struct {
71+
Name string `xml:"name,attr"`
72+
Flags string `xml:"flags,attr"`
73+
Epoch string `xml:"epoch,attr"`
74+
Ver string `xml:"ver,attr"`
75+
Rel string `xml:"rel,attr"`
76+
}
77+
78+
// Root structure of primary.xml metadata
79+
//
80+
// Contains a list of all packages available in the repository.
81+
type RPMPrimaryXML struct {
82+
Packages []RPMPackageXML `xml:"package"`
83+
}
84+
85+
// Additional metadata for an RPM package
86+
type RPMMeta struct {
87+
Checksum RPMChecksum
88+
Packager string
89+
Url string
90+
Format RPMFormat
91+
}

0 commit comments

Comments
 (0)