44 "context"
55 "encoding/json"
66 "io"
7- "log"
87 "net/http"
98 "os"
109 "slices"
@@ -15,6 +14,7 @@ import (
1514 ql "github.com/hasura/go-graphql-client"
1615 "github.com/terrapkg/gura/db"
1716 "github.com/terrapkg/gura/util"
17+ "go.uber.org/zap"
1818)
1919
2020// ? https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28#about-secondary-rate-limits
@@ -23,6 +23,7 @@ var pool_rest = make(chan db.Stream)
2323var pool_ql = make (chan db.Stream )
2424var tokens []Token
2525var tokensql []Token
26+ var ghl = util .SetupLog ("github" )
2627
2728type Token struct {
2829 key string
@@ -33,9 +34,7 @@ type Token struct {
3334func fetchGitHub (stream db.Stream ) {
3435 t , _ := util .SplitOnce (stream .Fetch , ' ' )
3536 i , err := strconv .ParseUint (t , 10 , 8 )
36- if err != nil {
37- log .Fatalf ("github: bad fetch [%s]: %v" , stream .Fetch , err )
38- }
37+ util .MaybeSuicide (ghl , "bad fetch" , err , zap .String ("stream.Fetch" , stream .Fetch ))
3938 switch GHFetchType (i ) {
4039 case TAG , QL :
4140 pool_ql <- stream
@@ -51,33 +50,17 @@ func fillGitHubTokens() *Token {
5150 num ++
5251 go func (token string ) {
5352 req , err := http .NewRequest (http .MethodGet , "https://api.github.com/rate_limit" , nil )
54- if err != nil {
55- log .Fatalln ("github: can't swim new req:" , err )
56- }
53+ util .MaybeSuicide (ghl , "can't swim new req" , err )
5754 req .Header .Add ("Authorization" , "Bearer " + token )
5855 resp , err := http .DefaultClient .Do (req )
59- if err != nil {
60- log .Printf ("github: GET /rate_limit: %v\n " +
61- "github: give up on token: %s" , err , token )
56+ if util .Yeet (ghl , "GET /rate_limit -> give up token" , err , zap .String ("token" , token )) {
6257 return
6358 }
64- quota , err := strconv .ParseInt (resp .Header .Get ("x-ratelimit-remaining" ), 10 , 16 )
65- if err != nil {
66- log .Printf ("github: strconv x-ratelimit-remaining: %v\n " +
67- "github: give up on token: %s" , err , token )
68- return
69- }
70- reset , err := strconv .ParseInt (resp .Header .Get ("x-ratelimit-reset" ), 10 , 64 )
71- if err != nil {
72- log .Printf ("github: strconv x-ratelimit-reset: %v\n " +
73- "github: give up on token: %s" , err , token )
74- return
75- }
76- token_chan <- Token {
77- key : token ,
78- quota : int16 (quota ),
79- reset : time .Unix (reset , 0 ),
59+ t := Token {
60+ key : token ,
8061 }
62+ updToken (& t , resp .Header )
63+ token_chan <- t
8164 }(token )
8265 }
8366 for ; num != 0 ; num -- {
@@ -96,7 +79,7 @@ func fillGitHubTokens() *Token {
9679 return + 1
9780 })
9881 if len (tokens ) == 0 {
99- log . Fatalln ( "github: fatal: no tokens"
82+ ghl . Panic ( " no tokens from GURA_GITHUB_TOKENS "
10083 }
10184 return & tokens [0 ]
10285}
@@ -116,13 +99,9 @@ func fillGitHubTokensQL() *Token {
11699 resetAt string
117100 }
118101 }
119- if err := qlcli .Query (context .Background (), & q , nil ); err != nil {
120- log .Fatalln ("github: can't query RateLimit for token" , token )
121- }
102+ util .MaybeSuicide (ghl , "can't query RateLimit" , qlcli .Query (context .Background (), & q , nil ), zap .String ("token" , token ))
122103 reset , err := time .Parse (time .RFC3339 , q .RateLimit .resetAt )
123- if err != nil {
124- log .Fatalln ("github: cannot parse time:" , q .RateLimit .resetAt )
125- }
104+ util .MaybeSuicide (ghl , "parse time" , err , zap .String ("resetAt" , q .RateLimit .resetAt ))
126105 token_chan <- Token {
127106 key : token ,
128107 quota : q .RateLimit .remaining ,
@@ -146,7 +125,7 @@ func fillGitHubTokensQL() *Token {
146125 return + 1
147126 })
148127 if len (tokens ) == 0 {
149- log . Fatalln ( "github: fatal: no tokens"
128+ ghl . Panic ( " no tokens from GURA_GITHUB_TOKENS "
150129 }
151130 return & tokensql [0 ]
152131}
@@ -160,20 +139,15 @@ func thanksForAllTheFish(token *Token, token_idx *int, tokens *[]Token) {
160139}
161140func waitForFish (token Token ) {
162141 if token .quota == 0 {
142+ ghl .Info ("wait for token reset" , zap .Time ("reset" , token .reset ))
163143 time .Sleep (time .Until (token .reset ))
164144 }
165145}
166146func updToken (token * Token , h http.Header ) {
167147 q , err := strconv .ParseInt (h .Get ("x-ratelimit-remaining" ), 10 , 16 )
168- if err != nil {
169- log .Fatalf ("github: strconv x-ratelimit-remaining: %v" , err )
170- return
171- }
148+ util .MaybeSuicide (ghl , "strconv x-ratelimit-remaining" , err )
172149 r , err := strconv .ParseInt (h .Get ("x-ratelimit-reset" ), 10 , 64 )
173- if err != nil {
174- log .Fatalf ("github: strconv x-ratelimit-reset: %v" , err )
175- return
176- }
150+ util .MaybeSuicide (ghl , "strconv x-ratelimit-reset" , err )
177151 token .quota = int16 (q )
178152 token .reset = time .Unix (r , 0 )
179153}
@@ -186,14 +160,13 @@ func swimGitHub() {
186160func swimGitHubRest () {
187161 for i , token := 0 , fillGitHubTokens (); noMoreFish (* token ); thanksForAllTheFish (token , & i , & tokens ) {
188162 waitForFish (* token )
189- for {
163+ for ! noMoreFish ( * token ) {
190164 stream := <- pool_rest
191- go func (stream db.Stream ) {
192- fetch (& stream , token , nil )
193- go schedule (stream )
194- db .DB .Save (stream )
195- }(stream )
165+ fetch (& stream , token , nil )
166+ go schedule (stream )
167+ go db .DB .Save (stream )
196168 }
169+ ghl .Info ("ran out of fish" , zap .Int ("token_idx" , i ))
197170 }
198171}
199172func swimGitHubQL () {
@@ -217,21 +190,17 @@ func swimGitHubQL() {
217190func rest_release (stream * db.Stream , remain string , token * Token ) {
218191 prefix , remain := util .SplitOnce (remain , ' ' )
219192 req , err := http .NewRequest (http .MethodGet , "https://api.github.com/repos/" + remain + "/releases" , nil )
220- if err != nil {
221- log .Printf ("github: req [%s]: %v" , stream .Fetch , err )
193+ if util .Yeet (ghl , "req fail" , err , zap .String ("fetch" , stream .Fetch )) {
222194 return
223195 }
224196 req .Header .Add ("Authorization" , "Bearer " + token .key )
225197 resp , err := http .DefaultClient .Do (req )
226- if err != nil {
227- log .Printf ("github: resp [%s]: %v" , stream .Fetch , err )
198+ if util .Yeet (ghl , "resp fail" , err , zap .String ("fetch" , stream .Fetch )) {
228199 return
229200 }
230201 updToken (token , resp .Header )
231202 buf , err := io .ReadAll (resp .Body )
232- if err != nil {
233- log .Fatalln ("github: can't read buf" )
234- }
203+ util .MaybeSuicide (ghl , "can't read buf" , err )
235204 var v []struct {
236205 tag_name string
237206 }
@@ -245,7 +214,7 @@ func rest_release(stream *db.Stream, remain string, token *Token) {
245214 return
246215 }
247216 }
248- log . Printf ( "github: [%s]: can't find prefix"stream .Fetch )
217+ ghl . Warn ( " can't find prefix"zap . String ( "fetch" , stream .Fetch ) )
249218}
250219
251220type GHFetchType = uint8
@@ -289,12 +258,11 @@ func ql_tag(stream *db.Stream, remain string, token *Token, qlcli ql.Client) {
289258 } `graphql:"repository(owner: $owner, name: $name)"`
290259 }
291260 var headers http.Header
292- if err := qlcli .Query (context .Background (), & q , map [string ]any {
261+ if util . Yeet ( ghl , "ql_tag" , qlcli .Query (context .Background (), & q , map [string ]any {
293262 "prefix" : prefix ,
294263 "owner" : owner ,
295264 "name" : name ,
296- }, ql .BindResponseHeaders (& headers )); err != nil {
297- log .Printf ("github: ql_tag: %v" , err )
265+ }, ql .BindResponseHeaders (& headers ))) {
298266 return
299267 }
300268 if v := strings .TrimPrefix (q .Repository .Refs .edges [0 ].node .name , prefix ); v != stream .Ver {
0 commit comments