Skip to content

Commit 4c0fdfd

Browse files
donhardmandonhardman
authored
Update to separate table per organization schema (#26)
Upgrade to use organization based table instead of global table for all indexed projects * Update to separate table per organization schema * Add two scripts for data migration for future use and showcase how we did it * Fix migration script 2 to group first before we add * Update composer deps * Implement fixes of known issues * Rename expansion_limit -> expansion_len due to interface change in Buddy * Change how we pass config to autocomplete method and handle it from client side * COnfigure CPU cores to use for each container * Remove usage of cpuset * Update config to use arm64 arch * Download arm64 version of the vector embedding extension * Build php cuz we extend it also * Update composer deps and fix issues with naming * Move manticore to separate server for a production use * Configure proper settings for deployment * Configure to use the single server for fpm and also for manticore daemon * Use amd64 PHP model extension * Download php model extension depending on architecture
1 parent 9f602f3 commit 4c0fdfd

31 files changed

+492
-226
lines changed

‎app/actions/api/autocomplete.php‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,16 @@
66
* @var string $org
77
* @var string $name
88
* @var string $query
9-
* @var array $config
9+
* @var int $fuzziness
10+
* @var int $expansion_len
11+
* @var bool $append
12+
* @var bool $prepend
13+
* @var array $layouts
1014
*/
1115

1216
use App\Component\Search;
1317

1418
/** @var App\Model\Repo $repo */
15-
[$org, $repo] = result(Search::getOrgAndRepo($org, $name));
16-
return Search::autocomplete($org, $repo, $query, $config);
19+
$org = result(Search::getOrg($org));
20+
$config = compact('fuzziness', 'expansion_len', 'append', 'prepend', 'layouts');
21+
return Search::autocomplete($org, $query, $config);

‎app/actions/api/repo.php‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,10 @@
99

1010
use App\Component\Search;
1111

12-
/** @var App\Model\Repo $repo */
13-
[$org, $repo] = result(Search::getOrgAndRepo($org, $name));
14-
return ok($repo ? $repo->toArray() : []);
12+
if ($name) {
13+
/** @var App\Model\Repo $repo */
14+
[$org, $repo] = result(Search::getOrgAndRepo($org, $name));
15+
return ok($repo ? $repo->toArray() : []);
16+
}
17+
18+
return ok([]);

‎app/actions/search.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
if ($query !== $search_query) {
2323
$show_query = true;
2424
}
25+
$filters['org_id'] = $org->id;
2526
if (!isset($filters['repos'])) {
2627
$filters['repos'] = $repo
2728
? [$repo->id]

‎app/client/component/search-form/index.js‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ export default element => {
1414
// Define default values
1515
const defaultConfig = {
1616
fuzziness: '2',
17-
expansion_limit: '6',
17+
expansion_len: '6',
1818
append: true,
1919
prepend: false,
2020
layouts: ['ru', 'us', 'ua'],
@@ -186,10 +186,10 @@ export default element => {
186186
for (const [key, value] of Object.entries(config)) {
187187
if (Array.isArray(value)) {
188188
value.forEach((item, index) => {
189-
body.append(`config[${key}][${index}]`, item);
189+
body.append(`${key}[${index}]`, item);
190190
});
191191
} else {
192-
body.append(`config[${key}]`, value);
192+
body.append(key, value);
193193
}
194194
}
195195

‎app/client/component/subscribe-form/index.js‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,22 @@ export default element => {
88
ev.preventDefault()
99
buttonEl.disabled = true
1010
const email = element.querySelector('input[type="text"]').value
11-
const response = await fetch(`${api_url}?email=${encodeURIComponent(email)}`)
12-
const [err, result] = await response.json()
13-
if (err) {
14-
alert("You entered an incorrect email. Please double-check it.")
15-
} else {
16-
alert('We will notify you once it\'s ready. Thanks!')
17-
}
11+
const response = await fetch(`${api_url}?email=${encodeURIComponent(email)}`)
1812

19-
buttonEl.disabled = false
13+
if (response.status !== 200) {
14+
alert("An error occurred. Please try again later.")
15+
buttonEl.disabled = false
16+
return
17+
}
18+
19+
const [err, result] = await response.json()
20+
if (err) {
21+
alert("You entered an incorrect email. Please double-check it.")
22+
} else {
23+
alert('We will notify you once it\'s ready. Thanks!')
24+
}
25+
26+
buttonEl.disabled = false
2027
})
2128

2229
return () => {}

‎app/composer.lock‎

Lines changed: 25 additions & 25 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎app/config/app.ini.tpl‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,10 +81,16 @@ credentials = 'true'
8181
host = 'queue'
8282
port = 4444
8383

84+
[queue:production]
85+
host = '10.0.0.4'
86+
8487
[manticore]
8588
host = 'manticore'
8689
port = 9308
8790

91+
[manticore:production]
92+
host = '10.0.0.4'
93+
8894
[github]
8995
organizations[] = 'manticoresoftware'
9096

‎app/core.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2249,6 +2249,8 @@ protected function block(string $key, mixed $param, mixed $item, Closure $block)
22492249
// If this is an object and array convertion exists
22502250
if (is_object($value) && method_exists($value, 'toArray')) {
22512251
$value = $value->toArray();
2252+
2253+
var_dump($value);
22522254
} elseif (!is_array($value)) {
22532255
$value = ['parent' => $item, 'this' => $value];
22542256
}

‎app/ruleset.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
<!-- Customizable PHP Codesniffer sniffs: https://github.com/squizlabs/PHP_CodeSniffer/wiki/Customisable-Sniff-Properties -->
33
<!-- Slevomat standard sniffs: https://github.com/slevomat/coding-standard#sniffs-included-in-this-standard -->
44
<ruleset name="muvon-codestyle">
5-
<description>A custom coding standard used by Muvon Co. Ltd projects</description>
5+
<description>A custom coding standard used by Manticore Software Co. Ltd projects</description>
66
<include-pattern>*\.php$</include-pattern>
77
<exclude-pattern>vendor/*</exclude-pattern>
88
<exclude-pattern>app/core.php</exclude-pattern>

‎app/scripts/data-migration-1‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#!/usr/bin/env /src/bin/php-exec-one
2+
<?php
3+
use Manticoresearch\Client;
4+
5+
ini_set('memory_limit', '512M');
6+
define('MANTICORE_CONFIG', array_merge(config('manticore'), [
7+
'retries' => 3,
8+
'timeout' => 86400,
9+
'connection_timeout' => 3,
10+
]));
11+
12+
$limit = 1000;
13+
$client = new Client(MANTICORE_CONFIG);
14+
$result = $client->sql('SELECT id, org_id FROM repo LIMIT 1000');
15+
$list = $result['hits']['hits'] ?? [];
16+
$org_map = [];
17+
foreach ($list as $item) {
18+
$id = $item['_id'];
19+
$org_id = $item['_source']['org_id'];
20+
$org_map[$org_id] ??= [];
21+
$org_map[$org_id][] = $id;
22+
}
23+
24+
foreach ($org_map as $org_id => $repo_ids) {
25+
Cli::print("Org id: $org_id");
26+
$repos = implode(',', $repo_ids);
27+
Cli::print("Repos: $repos");
28+
29+
Cli::print('Migrating issues');
30+
/* $client->sql('DROP TABLE IF EXISTS issue_' . $org_id, true); */
31+
$client->sql('CREATE TABLE IF NOT EXISTS issue_' . $org_id . ' LIKE issue WITH DATA', true);
32+
$client->sql('DELETE FROM issue_' . $org_id . ' WHERE repo_id NOT IN (' . $repos . ')', true);
33+
$client->sql('ALTER TABLE issue_' . $org_id . ' ADD COLUMN org_id bigint', true);
34+
$client->sql('UPDATE issue_' . $org_id . ' SET org_id = ' . $org_id . ' WHERE repo_id IN (' . $repos . ')', true);
35+
$client->sql('OPTIMIZE TABLE issue_' . $org_id . ' OPTION cutoff=1, sync=1', true);
36+
[$count] = $client->sql('SELECT COUNT(*) FROM issue_' . $org_id, true);
37+
Cli::print(" Count: $count");
38+
39+
Cli::print('Migrating comments');
40+
/* $client->sql('DROP TABLE IF EXISTS comment_' . $org_id, true); */
41+
$client->sql('CREATE TABLE IF NOT EXISTS comment_' . $org_id . ' LIKE `comment` WITH DATA', true);
42+
$client->sql('DELETE FROM comment_' . $org_id . ' WHERE repo_id NOT IN (' . $repos . ')', true);
43+
$client->sql('ALTER TABLE comment_' . $org_id . ' ADD COLUMN org_id bigint', true);
44+
$client->sql('UPDATE comment_' . $org_id . ' SET org_id = ' . $org_id . ' WHERE repo_id IN (' . $repos . ')', true);
45+
$client->sql('OPTIMIZE TABLE comment_' . $org_id . ' OPTION cutoff=1, sync=1', true);
46+
[$count] = $client->sql('SELECT COUNT(*) FROM comment_' . $org_id, true);
47+
Cli::print(" Count: $count");
48+
49+
}
50+
51+
Cli::print('Done');

0 commit comments

Comments
 (0)