Skip to content

Commit 92eff1e

Browse files
jerrinotjerrinot
authored
fix(sql): set default directory for read_parquet() and SQL COPY (#4819)
1 parent d0b618b commit 92eff1e

File tree

11 files changed

+98
-45
lines changed

11 files changed

+98
-45
lines changed

‎core/src/main/java/io/questdb/Bootstrap.java

Lines changed: 27 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -101,8 +101,9 @@ public Bootstrap(BootstrapConfiguration bootstrapConfiguration, String... args)
101101
}
102102

103103
// before we set up the logger, we need to copy the conf file
104+
byte[] buffer = new byte[1024 * 1024];
104105
try {
105-
copyLogConfResource(new byte[1024 * 1024]);
106+
copyLogConfResource(buffer);
106107
} catch (IOException e) {
107108
throw new BootstrapException("Could not extract log configuration file");
108109
}
@@ -114,6 +115,13 @@ public Bootstrap(BootstrapConfiguration bootstrapConfiguration, String... args)
114115
}
115116
log = LogFactory.getLog(LOG_NAME);
116117

118+
try {
119+
copyResource(rootDirectory, false, buffer, "import/readme.txt", log);
120+
copyResource(rootDirectory, false, buffer, "import/trades.parquet", log);
121+
} catch (IOException e) {
122+
throw new BootstrapException("Could not create the default import directory");
123+
}
124+
117125
// report copyright and architecture
118126
log.advisoryW()
119127
.$(buildInformation.getSwName()).$(' ').$(buildInformation.getSwVersion())
@@ -366,19 +374,6 @@ public CairoEngine newCairoEngine() {
366374
return new CairoEngine(getConfiguration().getCairoConfiguration(), getMetrics());
367375
}
368376

369-
private static void copyConfResource(String dir, boolean force, byte[] buffer, String res, Log log) throws IOException {
370-
copyConfResource(dir, force, buffer, res, res, log);
371-
}
372-
373-
private static void copyConfResource(String dir, boolean force, byte[] buffer, String res, String dest, Log log) throws IOException {
374-
File out = new File(dir, dest);
375-
try (InputStream is = ServerMain.class.getResourceAsStream("/io/questdb/site/" + res)) {
376-
if (is != null) {
377-
copyInputStream(force, buffer, out, is, log);
378-
}
379-
}
380-
}
381-
382377
private static void copyInputStream(boolean force, byte[] buffer, File out, InputStream is, Log log) throws IOException {
383378
final boolean exists = out.exists();
384379
if (force || !exists) {
@@ -405,6 +400,19 @@ private static void copyInputStream(boolean force, byte[] buffer, File out, Inpu
405400
}
406401
}
407402

403+
private static void copyResource(String dir, boolean force, byte[] buffer, String res, String dest, Log log) throws IOException {
404+
File out = new File(dir, dest);
405+
try (InputStream is = ServerMain.class.getResourceAsStream("/io/questdb/site/" + res)) {
406+
if (is != null) {
407+
copyInputStream(force, buffer, out, is, log);
408+
}
409+
}
410+
}
411+
412+
private static void copyResource(String dir, boolean force, byte[] buffer, String res, Log log) throws IOException {
413+
copyResource(dir, force, buffer, res, res, log);
414+
}
415+
408416
private static String getPublicVersion(String publicDir) throws IOException {
409417
File f = new File(publicDir, PUBLIC_VERSION_TXT);
410418
if (f.exists()) {
@@ -461,9 +469,9 @@ private static void verifyFileOpts(Path path, CairoConfiguration cairoConfigurat
461469

462470
private void copyLogConfResource(byte[] buffer) throws IOException {
463471
if (Chars.equalsIgnoreCaseNc("false", System.getProperty(CONTAINERIZED_SYSTEM_PROPERTY))) {
464-
copyConfResource(rootDirectory, false, buffer, "conf/non_containerized_log.conf", "conf/log.conf", null);
472+
copyResource(rootDirectory, false, buffer, "conf/non_containerized_log.conf", "conf/log.conf", null);
465473
} else {
466-
copyConfResource(rootDirectory, false, buffer, "conf/log.conf", null);
474+
copyResource(rootDirectory, false, buffer, "conf/log.conf", null);
467475
}
468476
}
469477

@@ -482,16 +490,16 @@ private void createHelloFile(String helloMsg) {
482490
}
483491

484492
private void extractConfDir(byte[] buffer) throws IOException {
485-
copyConfResource(rootDirectory, false, buffer, "conf/date.formats", log);
493+
copyResource(rootDirectory, false, buffer, "conf/date.formats", log);
486494
try {
487-
copyConfResource(rootDirectory, true, buffer, "conf/mime.types", log);
495+
copyResource(rootDirectory, true, buffer, "conf/mime.types", log);
488496
} catch (IOException exception) {
489497
// conf can be read-only, this is not critical
490498
if (exception.getMessage() == null || (!exception.getMessage().contains("Read-only file system") && !exception.getMessage().contains("Permission denied"))) {
491499
throw exception;
492500
}
493501
}
494-
copyConfResource(rootDirectory, false, buffer, "conf/server.conf", log);
502+
copyResource(rootDirectory, false, buffer, "conf/server.conf", log);
495503
copyLogConfResource(buffer);
496504
}
497505

‎core/src/main/java/io/questdb/PropServerConfiguration.java

Lines changed: 26 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -640,7 +640,8 @@ public PropServerConfiguration(
640640

641641
this.dbDirectory = getString(properties, env, PropertyKey.CAIRO_ROOT, DB_DIRECTORY);
642642
String tmpRoot;
643-
if (new File(this.dbDirectory).isAbsolute()) {
643+
boolean absDbDir = new File(this.dbDirectory).isAbsolute();
644+
if (absDbDir) {
644645
this.root = this.dbDirectory;
645646
this.confRoot = rootSubdir(this.root, CONFIG_DIRECTORY); // ../conf
646647
this.checkpointRoot = rootSubdir(this.root, TableUtils.CHECKPOINT_DIRECTORY); // ../.checkpoint
@@ -654,6 +655,30 @@ public PropServerConfiguration(
654655
tmpRoot = new File(root, TMP_DIRECTORY).getAbsolutePath();
655656
}
656657

658+
String configuredCairoSqlCopyRoot = getString(properties, env, PropertyKey.CAIRO_SQL_COPY_ROOT, "import");
659+
if (!Chars.empty(configuredCairoSqlCopyRoot)) {
660+
if (new File(configuredCairoSqlCopyRoot).isAbsolute()) {
661+
this.cairoSqlCopyRoot = configuredCairoSqlCopyRoot;
662+
} else {
663+
if (absDbDir) {
664+
this.cairoSqlCopyRoot = rootSubdir(this.root, configuredCairoSqlCopyRoot); // ../import
665+
} else {
666+
this.cairoSqlCopyRoot = new File(root, configuredCairoSqlCopyRoot).getAbsolutePath();
667+
}
668+
}
669+
String cairoSqlCopyWorkRoot = getString(properties, env, PropertyKey.CAIRO_SQL_COPY_WORK_ROOT, tmpRoot);
670+
this.cairoSqlCopyWorkRoot = getCanonicalPath(cairoSqlCopyWorkRoot);
671+
if (pathEquals(root, this.cairoSqlCopyWorkRoot)
672+
|| pathEquals(this.root, this.cairoSqlCopyWorkRoot)
673+
|| pathEquals(this.confRoot, this.cairoSqlCopyWorkRoot)
674+
|| pathEquals(this.checkpointRoot, this.cairoSqlCopyWorkRoot)) {
675+
throw new ServerConfigurationException("Configuration value for " + PropertyKey.CAIRO_SQL_COPY_WORK_ROOT.getPropertyPath() + " can't point to root, data, conf or snapshot dirs. ");
676+
}
677+
} else {
678+
this.cairoSqlCopyRoot = null;
679+
this.cairoSqlCopyWorkRoot = null;
680+
}
681+
657682
this.cairoAttachPartitionSuffix = getString(properties, env, PropertyKey.CAIRO_ATTACH_PARTITION_SUFFIX, TableUtils.ATTACHABLE_DIR_MARKER);
658683
this.cairoAttachPartitionCopy = getBoolean(properties, env, PropertyKey.CAIRO_ATTACH_PARTITION_COPY, false);
659684

@@ -1095,21 +1120,6 @@ public PropServerConfiguration(
10951120
inputFormatConfiguration.parseConfiguration(PropServerConfiguration.class, lexer, confRoot, sqlCopyFormatsFile);
10961121
}
10971122

1098-
this.cairoSqlCopyRoot = getString(properties, env, PropertyKey.CAIRO_SQL_COPY_ROOT, null);
1099-
String cairoSqlCopyWorkRoot = getString(properties, env, PropertyKey.CAIRO_SQL_COPY_WORK_ROOT, tmpRoot);
1100-
if (cairoSqlCopyRoot != null) {
1101-
this.cairoSqlCopyWorkRoot = getCanonicalPath(cairoSqlCopyWorkRoot);
1102-
} else {
1103-
this.cairoSqlCopyWorkRoot = null;
1104-
}
1105-
1106-
if (pathEquals(root, this.cairoSqlCopyWorkRoot)
1107-
|| pathEquals(this.root, this.cairoSqlCopyWorkRoot)
1108-
|| pathEquals(this.confRoot, this.cairoSqlCopyWorkRoot)
1109-
|| pathEquals(this.checkpointRoot, this.cairoSqlCopyWorkRoot)) {
1110-
throw new ServerConfigurationException("Configuration value for " + PropertyKey.CAIRO_SQL_COPY_WORK_ROOT.getPropertyPath() + " can't point to root, data, conf or snapshot dirs. ");
1111-
}
1112-
11131123
String cairoSQLCopyIdSupplier = getString(properties, env, PropertyKey.CAIRO_SQL_COPY_ID_SUPPLIER, "random");
11141124
this.cairoSQLCopyIdSupplier = Chars.equalsLowerCaseAscii(cairoSQLCopyIdSupplier, "sequential") ? 1 : 0;
11151125

‎core/src/main/java/io/questdb/ServerMain.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -348,7 +348,7 @@ protected void configureSharedPool(WorkerPool sharedPool) {
348348

349349
// text import
350350
CopyJob.assignToPool(engine.getMessageBus(), sharedPool);
351-
if (cairoConfig.getSqlCopyInputRoot() != null) {
351+
if (!Chars.empty(cairoConfig.getSqlCopyInputRoot())) {
352352
final CopyRequestJob copyRequestJob = new CopyRequestJob(
353353
engine,
354354
// save CPU resources for collecting and processing jobs

‎core/src/main/java/io/questdb/cairo/CairoConfiguration.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -373,7 +373,7 @@ default RostiAllocFacade getRostiAllocFacade() {
373373

374374
int getSqlCopyBufferSize();
375375

376-
// null input root disables "copy" sql
376+
// null or empty input root disables "copy" sql
377377
CharSequence getSqlCopyInputRoot();
378378

379379
CharSequence getSqlCopyInputWorkRoot();

‎core/src/main/resources/io/questdb/site/conf/server.conf

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -407,8 +407,9 @@ query.timeout.sec=60
407407
# name of file with user's set of date and timestamp formats
408408
#cairo.sql.copy.formats.file=/text_loader.json
409409

410-
# input root directory, where copy command reads files from
411-
#cairo.sql.copy.root=null
410+
# input root directory, where COPY command and read_parquet() function read files from
411+
# relative paths are resolved against the server root directory
412+
cairo.sql.copy.root=import
412413

413414
# input work directory, where temporary import files are created, by default it's located in tmp directory inside the server root directory
414415
#cairo.sql.copy.work.root=null

‎core/src/main/resources/io/questdb/site/import/readme.txt

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
This is the default directory for the SQL COPY command and the read_parquet() SQL function.
2+
3+
Drop files here to import data into QuestDB.
4+
5+
You can change the default directory by setting the cairo.sql.copy.root property in server.conf.
6+
7+
See:
8+
CSV import: https://questdb.io/docs/guides/import-csv/
9+
Reading external Parquet files: https://questdb.io/docs/reference/function/parquet/
10+
---
11+
12+
This directory also contains a demo parquet file: trades.parquet.
13+
You can use to test the read_parquet() SQL function.
14+
Open the QuestDB web console and run the following SQL query: SELECT * FROM read_parquet('trades.parquet');
15+
16+
You can run aggregation queries on the data in the Parquet file. For example:
17+
SELECT MIN(price), MAX(price)
18+
FROM read_parquet('trades.parquet');
19+
20+
Data in the demo Parquet file are sorted by timestamp. This allows for efficient time-series queries,
21+
but you need to specify the timestamp column when reading the Parquet file. For example, to sample the data by 1 day:
22+
WITH trades AS (
23+
(SELECT * FROM read_parquet('trades.parquet')) TIMESTAMP(timestamp)
24+
)
25+
SELECT timestamp, MIN(price), MAX(price), FIRST(price) AS open, LAST(price) AS close FROM trades
26+
SAMPLE BY 1d;
27+
28+
And last but not least, you can create a regular table and import the data from the Parquet file into it:
29+
CREATE TABLE trades AS
30+
(SELECT symbol::symbol AS symbol, side::symbol AS side, price, amount, timestamp FROM read_parquet('trades.parquet'))
31+
TIMESTAMP(timestamp)
32+
PARTITION BY MONTH WAL;

‎core/src/main/resources/io/questdb/site/import/trades.parquet

28.2 KB
Binary file not shown.

‎core/src/test/java/io/questdb/test/PropServerConfigurationTest.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -707,7 +707,7 @@ public void testImportWorkRootCantBeTheSameAsOtherInstanceDirectories() throws E
707707
Properties properties = new Properties();
708708

709709
PropServerConfiguration configuration = newPropServerConfiguration(root, properties, null, new BuildInformationHolder());
710-
Assert.assertNull(configuration.getCairoConfiguration().getSqlCopyInputWorkRoot());
710+
Assert.assertTrue(Chars.endsWith(configuration.getCairoConfiguration().getSqlCopyInputWorkRoot(), "tmp"));
711711

712712
//direct cases
713713
assertInputWorkRootCantBeSetTo(properties, root);
@@ -729,7 +729,8 @@ public void testImportWorkRootCantBeTheSameAsOtherInstanceDirectories2() throws
729729
Properties properties = new Properties();
730730

731731
PropServerConfiguration configuration = newPropServerConfiguration(root, properties, null, new BuildInformationHolder());
732-
Assert.assertNull(configuration.getCairoConfiguration().getSqlCopyInputWorkRoot());
732+
Assert.assertTrue(Chars.endsWith(configuration.getCairoConfiguration().getSqlCopyInputWorkRoot(), "tmp"));
733+
733734
assertInputWorkRootCantBeSetTo(properties, configuration.getCairoConfiguration().getRoot().toUpperCase());
734735
assertInputWorkRootCantBeSetTo(properties, configuration.getCairoConfiguration().getRoot().toLowerCase());
735736
}

‎core/src/test/java/io/questdb/test/cutlass/http/HttpErrorHandlingTest.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ public ServerConfiguration getServerConfiguration(Bootstrap bootstrap) throws Ex
6969
new FilesFacadeImpl() {
7070
@Override
7171
public long openRW(LPSZ name, long opts) {
72-
if (counter.incrementAndGet() > 28) {
72+
if (counter.incrementAndGet() > 69) {
7373
throw new RuntimeException("Test error");
7474
}
7575
return super.openRW(name, opts);

‎core/src/test/java/io/questdb/test/cutlass/pgwire/PGErrorHandlingTest.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ public ServerConfiguration getServerConfiguration(Bootstrap bootstrap) throws Ex
7070
new FilesFacadeImpl() {
7171
@Override
7272
public long openRW(LPSZ name, long opts) {
73-
if (counter.incrementAndGet() > 28) {
73+
if (counter.incrementAndGet() > 69) {
7474
throw new RuntimeException("Test error");
7575
}
7676
return super.openRW(name, opts);

‎pkg/ami/marketplace/assets/server.conf

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -379,8 +379,9 @@ query.timeout.sec=60
379379
# name of file with user's set of date and timestamp formats
380380
#cairo.sql.copy.formats.file=/text_loader.json
381381

382-
# input root directory, where copy command reads files from
383-
#cairo.sql.copy.root=null
382+
# input root directory, where COPY command and read_parquet() function read files from
383+
# relative paths are resolved against the server root directory
384+
cairo.sql.copy.root=import
384385

385386
# input work directory, where temporary import files are created, by default it's located in tmp directory inside the server root directory
386387
#cairo.sql.copy.work.root=null

0 commit comments

Comments
 (0)