Skip to content

Commit 07731dd

Browse files
madonukomadonuko
committed
feat(github): graphql for fetching tag
1 parent 950d6e9 commit 07731dd

File tree

5 files changed

+121
-17
lines changed

5 files changed

+121
-17
lines changed

‎go.mod‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ go 1.24.6
55
require (
66
github.com/gin-gonic/gin v1.10.1
77
github.com/google/uuid v1.6.0
8+
github.com/hasura/go-graphql-client v0.14.4
89
github.com/joho/godotenv v1.5.1
910
github.com/klauspost/compress v1.18.0
1011
gorm.io/datatypes v1.2.6
@@ -18,6 +19,7 @@ require (
1819
github.com/bytedance/sonic/loader v0.1.1 // indirect
1920
github.com/cloudwego/base64x v0.1.4 // indirect
2021
github.com/cloudwego/iasm v0.2.0 // indirect
22+
github.com/coder/websocket v1.8.13 // indirect
2123
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
2224
github.com/gin-contrib/sse v0.1.0 // indirect
2325
github.com/go-playground/locales v0.14.1 // indirect

‎go.sum‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/
88
github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
99
github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg=
1010
github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
11+
github.com/coder/websocket v1.8.13 h1:f3QZdXy7uGVz+4uCJy2nTZyM0yTBj8yANEHhqlXZ9FE=
12+
github.com/coder/websocket v1.8.13/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs=
1113
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
1214
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
1315
github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
@@ -30,6 +32,8 @@ github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MG
3032
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
3133
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
3234
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
35+
github.com/hasura/go-graphql-client v0.14.4 h1:bYU7/+V50T2YBGdNQXt6l4f2cMZPECPUd8cyCR+ixtw=
36+
github.com/hasura/go-graphql-client v0.14.4/go.mod h1:jfSZtBER3or+88Q9vFhWHiFMPppfYILRyl+0zsgPIIw=
3337
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
3438
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
3539
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=

‎nobori/github.go‎

Lines changed: 99 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package nobori
22

33
import (
4+
"context"
45
"log"
56
"net/http"
67
"os"
@@ -9,18 +10,27 @@ import (
910
"strings"
1011
"time"
1112

13+
ql "github.com/hasura/go-graphql-client"
1214
"github.com/terrapkg/gura/db"
15+
"github.com/terrapkg/gura/util"
1316
)
1417

1518
// ? https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28#about-secondary-rate-limits
1619
// no more than 100 parallel requests
1720
var pool = make(chan db.Stream, 100)
1821
var tokens []Token
22+
var ql_current_token Token
23+
var qlcli = ql.NewClient("https://api.github.com/graphql", http.DefaultClient).WithRequestModifier(
24+
func(r *http.Request) {
25+
r.Header.Add("Authorization", "Bearer"+ql_current_token.key)
26+
})
1927

2028
type Token struct {
2129
key string
2230
quota int16
2331
reset time.Time
32+
qlquo int16
33+
qlrst time.Time
2434
}
2535

2636
func fetchGitHub(stream db.Stream) {
@@ -100,12 +110,12 @@ func waitForFish(token Token) {
100110
func updToken(token *Token, resp *http.Response) {
101111
quota, err := strconv.ParseInt(resp.Header.Get("x-ratelimit-remaining"), 10, 16)
102112
if err != nil {
103-
log.Fatalf("github: strconv x-ratelimit-remaining: %v", err, *token)
113+
log.Fatalf("github: strconv x-ratelimit-remaining: %v", err)
104114
return
105115
}
106116
reset, err := strconv.ParseInt(resp.Header.Get("x-ratelimit-reset"), 10, 64)
107117
if err != nil {
108-
log.Fatalf("github: strconv x-ratelimit-reset: %v", err, *token)
118+
log.Fatalf("github: strconv x-ratelimit-reset: %v", err)
109119
return
110120
}
111121
token.quota = int16(quota)
@@ -119,19 +129,93 @@ func swimGitHub() {
119129
waitForFish(token)
120130
for {
121131
stream := <-pool
122-
req, err := http.NewRequest(http.MethodGet, "https://api.github.com/repos/"+stream.Fetch+"/tags", nil)
123-
if err != nil {
124-
log.Printf("github: req [%s]: %v", stream.Fetch, err)
125-
continue
126-
}
127-
req.Header.Add("Authorization", "Bearer "+token.key)
128-
resp, err := http.DefaultClient.Do(req)
129-
if err != nil {
130-
log.Printf("github: resp [%s]: %v", stream.Fetch, err)
131-
continue
132-
}
133-
updToken(&token, resp)
134-
// TODO: update stream
132+
go func(stream db.Stream) {
133+
fetch(&stream)
134+
go schedule(stream)
135+
db.DB.Save(stream)
136+
}(stream)
135137
}
136138
}
137139
}
140+
141+
/*
142+
func rest_tag(stream *db.Stream) {
143+
req, err := http.NewRequest(http.MethodGet, "https://api.github.com/repos/"+stream.Fetch+"/tags", nil)
144+
if err != nil {
145+
log.Printf("github: req [%s]: %v", stream.Fetch, err)
146+
continue
147+
}
148+
req.Header.Add("Authorization", "Bearer "+token.key)
149+
resp, err := http.DefaultClient.Do(req)
150+
if err != nil {
151+
log.Printf("github: resp [%s]: %v", stream.Fetch, err)
152+
continue
153+
}
154+
updToken(&token, resp)
155+
buf, err := io.ReadAll(resp.Body)
156+
if err != nil {
157+
log.Fatalln("github: can't read buf")
158+
}
159+
var v []struct {
160+
name string
161+
}
162+
json.Unmarshal(buf, &v)
163+
if stream.Ver != v[0].name {
164+
stream.Ver = v[0].name
165+
stream.LastUpd = time.Now()
166+
}
167+
}*/
168+
169+
type GHFetchType = int8
170+
171+
const (
172+
TAG GHFetchType = iota
173+
RELEASE
174+
QL
175+
)
176+
177+
// ? https://stackoverflow.com/questions/19452244/github-api-v3-order-tags-by-creation-date
178+
type TagQL struct {
179+
Repository struct {
180+
Refs struct {
181+
edges []struct {
182+
node struct {
183+
name string
184+
}
185+
}
186+
} `graphql:"refs(prefix: \"refs/tags/\", last: 1, orderBy: {field: TAG_COMMIT_DATE, direction: ASC}, before: $before), query: $prefix"`
187+
} `graphql:"repository(owner: $owner, name: $name)"`
188+
}
189+
190+
func fetch(stream *db.Stream) {
191+
s, remain := util.SplitOnce(stream.Fetch, ' ')
192+
i, err := strconv.Atoi(s)
193+
if err != nil {
194+
panic(err)
195+
}
196+
// TODO: support other GHFetchType
197+
switch GHFetchType(i) {
198+
case TAG:
199+
ql_tag(stream, remain)
200+
}
201+
stream.LastChk = time.Now()
202+
}
203+
204+
func ql_tag(stream *db.Stream, remain string) {
205+
prefix, remain := util.SplitOnce(remain, ' ')
206+
owner, name := util.SplitOnce(remain, '/')
207+
var q TagQL
208+
if err := qlcli.Query(context.Background(), &q, map[string]any{
209+
"prefix": prefix,
210+
"before": nil,
211+
"owner": owner,
212+
"name": name,
213+
}); err != nil {
214+
log.Printf("github: ql_tag: %v", err)
215+
return
216+
}
217+
if v := strings.TrimPrefix(q.Repository.Refs.edges[0].node.name, prefix); v != stream.Ver {
218+
stream.Ver = v
219+
stream.LastUpd = time.Now()
220+
}
221+
}

‎nobori/nobori.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ func schedule(stream db.Stream) {
3939
}
4040

4141
func FetchLoop() {
42+
go swimGitHub()
4243
var streams []db.Stream
4344
if r := db.DB.Find(&streams); r.Error != nil {
4445
log.Fatalln("nobori: fatal:", r.Error)
@@ -48,8 +49,8 @@ func FetchLoop() {
4849
}
4950
for {
5051
switch stream := <-queue; stream.Forge {
51-
case db.GitHub:
52-
go fetchGitHub(stream)
52+
case db.GitHub:
53+
go fetchGitHub(stream)
5354
}
5455
}
5556
}

‎util/str.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package util
2+
3+
func SplitOnce(s string, c byte) (left string, right string) {
4+
for i := 0; i < len(s); i++ {
5+
if s[i] == c {
6+
left = s[0:i]
7+
right = s[i+1:]
8+
return
9+
}
10+
}
11+
left = s
12+
return
13+
}

0 commit comments

Comments
 (0)