Skip to content

Commit 3898066

Browse files
stainless-app[bot]stainless-app[bot]
committed
chore: refactor request options
1 parent 3a18c70 commit 3898066

File tree

4 files changed

+128
-111
lines changed

4 files changed

+128
-111
lines changed

‎src/Client.php‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44

55
namespace Scrapegraphai;
66

7+
use Http\Discovery\Psr17FactoryDiscovery;
8+
use Http\Discovery\Psr18ClientDiscovery;
79
use Scrapegraphai\Core\BaseClient;
810
use Scrapegraphai\Core\Services\CrawlService;
911
use Scrapegraphai\Core\Services\CreditsService;
@@ -72,12 +74,19 @@ public function __construct(?string $apiKey = null, ?string $baseUrl = null)
7274
'SCRAPEGRAPHAI_BASE_URL'
7375
) ?: 'https://api.scrapegraphai.com/v1';
7476

77+
$options = new RequestOptions(
78+
uriFactory: Psr17FactoryDiscovery::findUriFactory(),
79+
streamFactory: Psr17FactoryDiscovery::findStreamFactory(),
80+
requestFactory: Psr17FactoryDiscovery::findRequestFactory(),
81+
transporter: Psr18ClientDiscovery::find(),
82+
);
83+
7584
parent::__construct(
7685
headers: [
7786
'Content-Type' => 'application/json', 'Accept' => 'application/json',
7887
],
7988
baseUrl: $base,
80-
options: new RequestOptions,
89+
options: $options,
8190
);
8291

8392
$this->smartscraper = new SmartscraperService($this);

‎src/Core/BaseClient.php‎

Lines changed: 23 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,6 @@
44

55
namespace Scrapegraphai\Core;
66

7-
use Http\Discovery\Psr17FactoryDiscovery;
8-
use Http\Discovery\Psr18ClientDiscovery;
97
use Psr\Http\Client\ClientInterface;
108
use Psr\Http\Message\RequestFactoryInterface;
119
use Psr\Http\Message\RequestInterface;
@@ -33,14 +31,6 @@ class BaseClient
3331
{
3432
protected UriInterface $baseUrl;
3533

36-
protected UriFactoryInterface $uriFactory;
37-
38-
protected StreamFactoryInterface $streamFactory;
39-
40-
protected RequestFactoryInterface $requestFactory;
41-
42-
protected ClientInterface $transporter;
43-
4434
/**
4535
* @param array<string, string|int|list<string|int>|null> $headers
4636
*/
@@ -49,12 +39,8 @@ public function __construct(
4939
string $baseUrl,
5040
protected RequestOptions $options = new RequestOptions,
5141
) {
52-
$this->uriFactory = Psr17FactoryDiscovery::findUriFactory();
53-
$this->streamFactory = Psr17FactoryDiscovery::findStreamFactory();
54-
$this->requestFactory = Psr17FactoryDiscovery::findRequestFactory();
55-
56-
$this->baseUrl = $this->uriFactory->createUri($baseUrl);
57-
$this->transporter = Psr18ClientDiscovery::find();
42+
assert(null !== $this->options->uriFactory);
43+
$this->baseUrl = $this->options->uriFactory->createUri($baseUrl);
5844
}
5945

6046
/**
@@ -63,6 +49,7 @@ public function __construct(
6349
* @param array<string, mixed> $headers
6450
* @param class-string<BasePage<mixed>> $page
6551
* @param class-string<BaseStream<mixed>> $stream
52+
* @param RequestOptions|array<string, mixed>|null $options
6653
*/
6754
public function request(
6855
string $method,
@@ -73,17 +60,18 @@ public function request(
7360
string|Converter|ConverterSource|null $convert = null,
7461
?string $page = null,
7562
?string $stream = null,
76-
mixed $options = [],
63+
RequestOptions|array|null $options = [],
7764
): mixed {
7865
// @phpstan-ignore-next-line
7966
[$req, $opts] = $this->buildRequest(method: $method, path: $path, query: $query, headers: $headers, body: $body, opts: $options);
8067
['method' => $method, 'path' => $uri, 'headers' => $headers] = $req;
68+
assert(null !== $opts->requestFactory);
8169

82-
$request = $this->requestFactory->createRequest($method, uri: $uri);
70+
$request = $opts->requestFactory->createRequest($method, uri: $uri);
8371
$request = Util::withSetHeaders($request, headers: $headers);
8472

8573
// @phpstan-ignore-next-line
86-
$rsp = $this->sendRequest($request, data: $body, opts: $opts, redirectCount: 0, retryCount: 0);
74+
$rsp = $this->sendRequest($opts, req: $request, data: $body, redirectCount: 0, retryCount: 0);
8775

8876
$decoded = Util::decodeContent($rsp);
8977

@@ -123,14 +111,18 @@ protected function authHeaders(): array
123111
* @param string|list<string> $path
124112
* @param array<string, mixed> $query
125113
* @param array<string, string|int|list<string|int>|null> $headers
126-
* @param RequestOptions|array{
114+
* @param array{
127115
* timeout?: float|null,
128116
* maxRetries?: int|null,
129117
* initialRetryDelay?: float|null,
130118
* maxRetryDelay?: float|null,
131-
* extraHeaders?: list<string>|null,
132-
* extraQueryParams?: list<string>|null,
133-
* extraBodyParams?: list<string>|null,
119+
* extraHeaders?: array<string, string|int|list<string|int>|null>|null,
120+
* extraQueryParams?: array<string, mixed>|null,
121+
* extraBodyParams?: mixed,
122+
* transporter?: ClientInterface|null,
123+
* uriFactory?: UriFactoryInterface|null,
124+
* streamFactory?: StreamFactoryInterface|null,
125+
* requestFactory?: RequestFactoryInterface|null,
134126
* }|null $opts
135127
*
136128
* @return array{normalized_request, RequestOptions}
@@ -143,7 +135,7 @@ protected function buildRequest(
143135
mixed $body,
144136
RequestOptions|array|null $opts,
145137
): array {
146-
$opts = [...$this->options->__serialize(), ...RequestOptions::parse($opts)->__serialize()];
138+
$opts = array_merge($this->options->toArray(), RequestOptions::parse($opts)->toArray());
147139
$options = new RequestOptions(...$opts);
148140

149141
$parsedPath = Util::parsePath($path);
@@ -182,14 +174,16 @@ protected function followRedirect(
182174
* mixed,>|null $data
183175
*/
184176
protected function sendRequest(
177+
RequestOptions $opts,
185178
RequestInterface $req,
186179
mixed $data,
187-
RequestOptions $opts,
188180
int $retryCount,
189181
int $redirectCount,
190182
): ResponseInterface {
191-
$req = Util::withSetBody($this->streamFactory, req: $req, body: $data);
192-
$rsp = $this->transporter->sendRequest($req);
183+
assert(null !== $opts->streamFactory && null !== $opts->transporter);
184+
185+
$req = Util::withSetBody($opts->streamFactory, req: $req, body: $data);
186+
$rsp = $opts->transporter->sendRequest($req);
193187
$code = $rsp->getStatusCode();
194188

195189
if ($code >= 300 && $code < 400) {
@@ -199,7 +193,7 @@ protected function sendRequest(
199193

200194
$req = $this->followRedirect($rsp, req: $req);
201195

202-
return $this->sendRequest($req, data: $data, opts: $opts, retryCount: $retryCount, redirectCount: ++$redirectCount);
196+
return $this->sendRequest($opts, req: $req, data: $data, retryCount: $retryCount, redirectCount: ++$redirectCount);
203197
}
204198

205199
if ($code >= 400 && $code < 500) {
@@ -209,7 +203,7 @@ protected function sendRequest(
209203
if ($code >= 500 && $retryCount < $opts->maxRetries) {
210204
usleep((int) $opts->initialRetryDelay);
211205

212-
return $this->sendRequest($req, data: $data, opts: $opts, retryCount: ++$retryCount, redirectCount: $redirectCount);
206+
return $this->sendRequest($opts, req: $req, data: $data, retryCount: ++$retryCount, redirectCount: $redirectCount);
213207
}
214208

215209
return $rsp;

‎src/Core/Contracts/BasePage.php‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
/**
1313
* @internal
1414
*
15+
* @phpstan-import-type normalized_request from \Scrapegraphai\Core\BaseClient
16+
*
1517
* @template Item
1618
*
1719
* @extends \IteratorAggregate<int, static>
@@ -21,7 +23,7 @@ interface BasePage extends \IteratorAggregate
2123
/**
2224
* @internal
2325
*
24-
* @param array<string, mixed> $request
26+
* @param normalized_request $request
2527
*/
2628
public function __construct(
2729
Converter|ConverterSource|string $convert,

‎src/RequestOptions.php‎

Lines changed: 92 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -4,99 +4,111 @@
44

55
namespace Scrapegraphai;
66

7-
class RequestOptions
7+
use Psr\Http\Client\ClientInterface;
8+
use Psr\Http\Message\RequestFactoryInterface;
9+
use Psr\Http\Message\StreamFactoryInterface;
10+
use Psr\Http\Message\UriFactoryInterface;
11+
use Scrapegraphai\Core\Attributes\Api as Property;
12+
use Scrapegraphai\Core\Concerns\SdkModel;
13+
use Scrapegraphai\Core\Contracts\BaseModel;
14+
use Scrapegraphai\Core\Implementation\Omittable;
15+
16+
use const Scrapegraphai\Core\OMIT as omit;
17+
18+
/**
19+
* @phpstan-type request_options = array{
20+
* timeout?: float|null,
21+
* maxRetries?: int|null,
22+
* initialRetryDelay?: float|null,
23+
* maxRetryDelay?: float|null,
24+
* extraHeaders?: array<string, string|int|null|list<string|int>>|null,
25+
* extraQueryParams?: array<string, mixed>|null,
26+
* extraBodyParams?: mixed,
27+
* transporter?: ClientInterface|null,
28+
* uriFactory?: UriFactoryInterface|null,
29+
* streamFactory?: StreamFactoryInterface|null,
30+
* requestFactory?: RequestFactoryInterface|null,
31+
* }
32+
* @phpstan-type request_opts = null|RequestOptions|request_options
33+
*/
34+
final class RequestOptions implements BaseModel
835
{
9-
public const DEFAULT_TIMEOUT = 60;
36+
/** @use SdkModel<request_options> */
37+
use SdkModel;
1038

11-
public const DEFAULT_MAX_RETRIES = 2;
39+
#[Property]
40+
public float $timeout = 60;
1241

13-
public const DEFAULT_INITIAL_RETRYDELAY = 0.5;
42+
#[Property]
43+
public int $maxRetries = 2;
1444

15-
public const DEFAULT_MAX_RETRY_DELAY = 8.0;
45+
#[Property]
46+
public float $initialRetryDelay = 0.5;
1647

17-
/**
18-
* @param list<string> $extraHeaders
19-
* @param list<string> $extraQueryParams
20-
* @param list<string> $extraBodyParams
21-
*/
22-
public function __construct(
23-
public float $timeout = self::DEFAULT_TIMEOUT,
24-
public int $maxRetries = self::DEFAULT_MAX_RETRIES,
25-
public float $initialRetryDelay = self::DEFAULT_INITIAL_RETRYDELAY,
26-
public float $maxRetryDelay = self::DEFAULT_MAX_RETRY_DELAY,
27-
public array $extraHeaders = [],
28-
public array $extraQueryParams = [],
29-
public array $extraBodyParams = [],
30-
) {}
48+
#[Property]
49+
public float $maxRetryDelay = 8.0;
3150

32-
/**
33-
* @return array{
34-
* timeout: float,
35-
* maxRetries: int,
36-
* initialRetryDelay: float,
37-
* maxRetryDelay: float,
38-
* extraHeaders: list<string>,
39-
* extraQueryParams: list<string>,
40-
* extraBodyParams: list<string>,
41-
* }
42-
*/
43-
public function __serialize(): array
44-
{
45-
return [
46-
'timeout' => $this->timeout,
47-
'maxRetries' => $this->maxRetries,
48-
'initialRetryDelay' => $this->initialRetryDelay,
49-
'maxRetryDelay' => $this->maxRetryDelay,
50-
'extraHeaders' => $this->extraHeaders,
51-
'extraQueryParams' => $this->extraQueryParams,
52-
'extraBodyParams' => $this->extraBodyParams,
53-
];
54-
}
51+
/** @var array<string, string|int|list<string|int>|null> $extraHeaders */
52+
#[Property]
53+
public array $extraHeaders = [];
54+
55+
/** @var array<string, mixed> $extraQueryParams */
56+
#[Property]
57+
public array $extraQueryParams = [];
58+
59+
#[Property]
60+
public mixed $extraBodyParams;
61+
62+
#[Property(optional: true)]
63+
public ?ClientInterface $transporter;
64+
65+
#[Property(optional: true)]
66+
public ?UriFactoryInterface $uriFactory;
67+
68+
#[Property(optional: true)]
69+
public ?StreamFactoryInterface $streamFactory;
70+
71+
#[Property(optional: true)]
72+
public ?RequestFactoryInterface $requestFactory;
5573

5674
/**
57-
* @param array{
58-
* timeout?: float|null,
59-
* maxRetries?: int|null,
60-
* initialRetryDelay?: float|null,
61-
* maxRetryDelay?: float|null,
62-
* extraHeaders?: list<string>|null,
63-
* extraQueryParams?: list<string>|null,
64-
* extraBodyParams?: list<string>|null,
65-
* } $data
75+
* @param array<string, string|int|list<string|int>|null>|null $extraHeaders
76+
* @param array<string, mixed>|null $extraQueryParams
77+
* @param mixed|Omittable $extraBodyParams
6678
*/
67-
public function __unserialize(array $data): void
68-
{
69-
$this->timeout = $data['timeout'] ?? self::DEFAULT_TIMEOUT;
70-
$this
71-
->maxRetries = $data['maxRetries'] ?? self::DEFAULT_MAX_RETRIES
72-
;
73-
$this
74-
->initialRetryDelay = $data[
75-
'initialRetryDelay'
76-
] ?? self::DEFAULT_INITIAL_RETRYDELAY
77-
;
78-
$this->maxRetryDelay = $data[
79-
'maxRetryDelay'
80-
] ?? self::DEFAULT_MAX_RETRY_DELAY;
81-
$this->extraHeaders = $data[
82-
'extraHeaders'
83-
] ?? [];
84-
$this->extraQueryParams = $data['extraQueryParams'] ?? [];
85-
$this
86-
->extraBodyParams = $data['extraBodyParams'] ?? []
79+
public function __construct(
80+
?float $timeout = null,
81+
?int $maxRetries = null,
82+
?float $initialRetryDelay = null,
83+
?float $maxRetryDelay = null,
84+
?array $extraHeaders = null,
85+
?array $extraQueryParams = null,
86+
mixed $extraBodyParams = omit,
87+
?ClientInterface $transporter = null,
88+
?UriFactoryInterface $uriFactory = null,
89+
?StreamFactoryInterface $streamFactory = null,
90+
?RequestFactoryInterface $requestFactory = null,
91+
) {
92+
self::introspect();
93+
$this->unsetOptionalProperties();
94+
95+
null !== $timeout && $this->timeout = $timeout;
96+
null !== $maxRetries && $this->maxRetries = $maxRetries;
97+
null !== $initialRetryDelay && $this
98+
->initialRetryDelay = $initialRetryDelay
8799
;
100+
null !== $maxRetryDelay && $this->maxRetryDelay = $maxRetryDelay;
101+
null !== $extraHeaders && $this->extraHeaders = $extraHeaders;
102+
null !== $extraQueryParams && $this->extraQueryParams = $extraQueryParams;
103+
omit !== $extraBodyParams && $this->extraBodyParams = $extraBodyParams;
104+
null !== $transporter && $this->transporter = $transporter;
105+
null !== $uriFactory && $this->uriFactory = $uriFactory;
106+
null !== $streamFactory && $this->streamFactory = $streamFactory;
107+
null !== $requestFactory && $this->requestFactory = $requestFactory;
88108
}
89109

90110
/**
91-
* @param RequestOptions|array{
92-
* timeout?: float|null,
93-
* maxRetries?: int|null,
94-
* initialRetryDelay?: float|null,
95-
* maxRetryDelay?: float|null,
96-
* extraHeaders?: list<string>|null,
97-
* extraQueryParams?: list<string>|null,
98-
* extraBodyParams?: list<string>|null,
99-
* }|null $options
111+
* @param request_opts|null $options
100112
*/
101113
public static function parse(RequestOptions|array|null $options): self
102114
{

0 commit comments

Comments
 (0)