Skip to content

Commit db190b4

Browse files
author
Tianhua Ran
committed
refactor: speed up maintaining database
1 parent 4601b76 commit db190b4

File tree

2 files changed

+153
-114
lines changed

2 files changed

+153
-114
lines changed

‎library/src/main/java/com/liulishuo/filedownloader/database/SqliteDatabaseImpl.java‎

Lines changed: 27 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -35,22 +35,21 @@
3535

3636
/**
3737
* Persist data to SQLite database.
38-
*
38+
* <p>
3939
* You can valid this database implementation through:
4040
* <p>
4141
* class MyApplication extends Application {
4242
* ...
4343
* public void onCreate() {
44-
* ...
45-
* FileDownloader.setupOnApplicationOnCreate(this)
46-
* .database(SqliteDatabaseImpl.createMaker())
47-
* ...
48-
* .commit();
49-
* ...
44+
* ...
45+
* FileDownloader.setupOnApplicationOnCreate(this)
46+
* .database(SqliteDatabaseImpl.createMaker())
47+
* ...
48+
* .commit();
49+
* ...
5050
* }
5151
* ...
5252
* }
53-
*
5453
*/
5554
public class SqliteDatabaseImpl implements FileDownloadDatabase {
5655

@@ -267,6 +266,7 @@ private void update(final int id, final ContentValues cv) {
267266
public class Maintainer implements FileDownloadDatabase.Maintainer {
268267

269268
private final SparseArray<FileDownloadModel> needChangeIdList = new SparseArray<>();
269+
private final SparseArray<FileDownloadModel> needRemoveList = new SparseArray<>();
270270
private MaintainerIterator currentIterator;
271271

272272
private final SparseArray<FileDownloadModel> downloaderModelMap;
@@ -317,6 +317,14 @@ public void onFinishMaintain() {
317317
}
318318
}
319319

320+
// remove invalid
321+
final int removeSize = needRemoveList.size();
322+
for (int i = 0; i < removeSize; i++) {
323+
final int modelId = needRemoveList.keyAt(i);
324+
db.delete(TABLE_NAME, FileDownloadModel.ID + " = ?",
325+
new String[]{String.valueOf(modelId)});
326+
}
327+
320328
// initial cache of connection model
321329
if (downloaderModelMap != null && connectionModelListMap != null) {
322330
final int size = downloaderModelMap.size();
@@ -339,16 +347,25 @@ public void onFinishMaintain() {
339347

340348
@Override
341349
public void onRemovedInvalidData(FileDownloadModel model) {
350+
synchronized (needRemoveList) {
351+
needRemoveList.put(model.getId(), model);
352+
}
342353
}
343354

344355
@Override
345356
public void onRefreshedValidData(FileDownloadModel model) {
346-
if (downloaderModelMap != null) downloaderModelMap.put(model.getId(), model);
357+
if (downloaderModelMap != null) {
358+
synchronized (downloaderModelMap) {
359+
downloaderModelMap.put(model.getId(), model);
360+
}
361+
}
347362
}
348363

349364
@Override
350365
public void changeFileDownloadModelId(int oldId, FileDownloadModel modelWithNewId) {
351-
needChangeIdList.put(oldId, modelWithNewId);
366+
synchronized (needChangeIdList) {
367+
needChangeIdList.put(oldId, modelWithNewId);
368+
}
352369
}
353370

354371
}

‎library/src/main/java/com/liulishuo/filedownloader/download/CustomComponentHolder.java‎

Lines changed: 126 additions & 104 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,19 @@
2323
import com.liulishuo.filedownloader.services.DownloadMgrInitialParams;
2424
import com.liulishuo.filedownloader.services.ForegroundServiceConfig;
2525
import com.liulishuo.filedownloader.stream.FileDownloadOutputStream;
26+
import com.liulishuo.filedownloader.util.FileDownloadExecutors;
2627
import com.liulishuo.filedownloader.util.FileDownloadHelper;
2728
import com.liulishuo.filedownloader.util.FileDownloadLog;
2829
import com.liulishuo.filedownloader.util.FileDownloadUtils;
2930

3031
import java.io.File;
3132
import java.io.IOException;
33+
import java.util.ArrayList;
3234
import java.util.Iterator;
35+
import java.util.List;
36+
import java.util.concurrent.Future;
37+
import java.util.concurrent.ThreadPoolExecutor;
38+
import java.util.concurrent.atomic.AtomicInteger;
3339

3440
/**
3541
* The holder for supported custom components.
@@ -88,17 +94,7 @@ public FileDownloadDatabase getDatabaseInstance() {
8894
synchronized (this) {
8995
if (database == null) {
9096
database = getDownloadMgrInitialParams().createDatabase();
91-
// There is no reusable thread for this action and this action has
92-
// a very low frequency, so, a new thread is simplest way to make this action
93-
// run on background thread.
94-
final String maintainThreadName =
95-
FileDownloadUtils.getThreadPoolName("MaintainDatabase");
96-
new Thread(new Runnable() {
97-
@Override
98-
public void run() {
99-
maintainDatabase(database.maintainer());
100-
}
101-
}, maintainThreadName).start();
97+
maintainDatabase(database.maintainer());
10298
}
10399
}
104100

@@ -178,123 +174,149 @@ private DownloadMgrInitialParams getDownloadMgrInitialParams() {
178174
return initialParams;
179175
}
180176

181-
private static void maintainDatabase(FileDownloadDatabase.Maintainer maintainer) {
177+
private static void maintainDatabase(final FileDownloadDatabase.Maintainer maintainer) {
182178
final Iterator<FileDownloadModel> iterator = maintainer.iterator();
183-
long refreshDataCount = 0;
184-
long removedDataCount = 0;
185-
long resetIdCount = 0;
179+
final AtomicInteger removedDataCount = new AtomicInteger(0);
180+
final AtomicInteger resetIdCount = new AtomicInteger(0);
181+
final AtomicInteger refreshDataCount = new AtomicInteger(0);
186182
final FileDownloadHelper.IdGenerator idGenerator = getImpl().getIdGeneratorInstance();
187183

188184
final long startTimestamp = System.currentTimeMillis();
185+
final List<Future> futures = new ArrayList<>();
186+
final ThreadPoolExecutor maintainThreadPool = FileDownloadExecutors.newDefaultThreadPool(3,
187+
FileDownloadUtils.getThreadPoolName("MaintainDatabase"));
189188
try {
190189
while (iterator.hasNext()) {
191-
boolean isInvalid = false;
192190
final FileDownloadModel model = iterator.next();
193-
do {
194-
if (model.getStatus() == FileDownloadStatus.progress
195-
|| model.getStatus() == FileDownloadStatus.connected
196-
|| model.getStatus() == FileDownloadStatus.error
197-
|| (model.getStatus() == FileDownloadStatus.pending && model
198-
.getSoFar() > 0)
199-
) {
200-
// Ensure can be covered by RESUME FROM BREAKPOINT.
201-
model.setStatus(FileDownloadStatus.paused);
202-
}
203-
final String targetFilePath = model.getTargetFilePath();
204-
if (targetFilePath == null) {
205-
// no target file path, can't used to resume from breakpoint.
206-
isInvalid = true;
207-
break;
208-
}
209-
210-
final File targetFile = new File(targetFilePath);
211-
// consider check in new thread, but SQLite lock | file lock aways effect, so
212-
// sync
213-
if (model.getStatus() == FileDownloadStatus.paused
214-
&& FileDownloadUtils.isBreakpointAvailable(model.getId(), model,
215-
model.getPath(), null)) {
216-
// can be reused in the old mechanism(no-temp-file).
217-
218-
final File tempFile = new File(model.getTempFilePath());
191+
final Future modelFuture = maintainThreadPool.submit(new Runnable() {
192+
@Override
193+
public void run() {
194+
boolean isInvalid = false;
195+
do {
196+
if (model.getStatus() == FileDownloadStatus.progress
197+
|| model.getStatus() == FileDownloadStatus.connected
198+
|| model.getStatus() == FileDownloadStatus.error
199+
|| (model.getStatus() == FileDownloadStatus.pending && model
200+
.getSoFar() > 0)
201+
) {
202+
// Ensure can be covered by RESUME FROM BREAKPOINT.
203+
model.setStatus(FileDownloadStatus.paused);
204+
}
205+
final String targetFilePath = model.getTargetFilePath();
206+
if (targetFilePath == null) {
207+
// no target file path, can't used to resume from breakpoint.
208+
isInvalid = true;
209+
break;
210+
}
219211

220-
if (!tempFile.exists() && targetFile.exists()) {
221-
final boolean successRename = targetFile.renameTo(tempFile);
222-
if (FileDownloadLog.NEED_LOG) {
223-
FileDownloadLog.d(FileDownloadDatabase.class,
224-
"resume from the old no-temp-file architecture "
225-
+ "[%B], [%s]->[%s]",
226-
successRename, targetFile.getPath(), tempFile.getPath());
212+
final File targetFile = new File(targetFilePath);
213+
// consider check in new thread, but SQLite lock | file lock aways
214+
// effect, so sync
215+
if (model.getStatus() == FileDownloadStatus.paused
216+
&& FileDownloadUtils.isBreakpointAvailable(model.getId(), model,
217+
model.getPath(), null)) {
218+
// can be reused in the old mechanism(no-temp-file).
219+
220+
final File tempFile = new File(model.getTempFilePath());
221+
222+
if (!tempFile.exists() && targetFile.exists()) {
223+
final boolean successRename = targetFile.renameTo(tempFile);
224+
if (FileDownloadLog.NEED_LOG) {
225+
FileDownloadLog.d(FileDownloadDatabase.class,
226+
"resume from the old no-temp-file architecture "
227+
+ "[%B], [%s]->[%s]",
228+
successRename, targetFile.getPath(),
229+
tempFile.getPath());
230+
}
231+
}
232+
}
227233

234+
/**
235+
* Remove {@code model} from DB if it can't used for judging whether the
236+
* old-downloaded file is valid for reused & it can't used for resuming
237+
* from BREAKPOINT, In other words, {@code model} is no use anymore for
238+
* FileDownloader.
239+
*/
240+
if (model.getStatus() == FileDownloadStatus.pending
241+
&& model.getSoFar() <= 0) {
242+
// This model is redundant.
243+
isInvalid = true;
244+
break;
228245
}
229-
}
230-
}
231246

232-
/**
233-
* Remove {@code model} from DB if it can't used for judging whether the
234-
* old-downloaded file is valid for reused & it can't used for resuming from
235-
* BREAKPOINT, In other words, {@code model} is no use anymore for
236-
* FileDownloader.
237-
*/
238-
if (model.getStatus() == FileDownloadStatus.pending && model.getSoFar() <= 0) {
239-
// This model is redundant.
240-
isInvalid = true;
241-
break;
242-
}
247+
if (!FileDownloadUtils.isBreakpointAvailable(model.getId(), model)) {
248+
// It can't used to resuming from breakpoint.
249+
isInvalid = true;
250+
break;
251+
}
243252

244-
if (!FileDownloadUtils.isBreakpointAvailable(model.getId(), model)) {
245-
// It can't used to resuming from breakpoint.
246-
isInvalid = true;
247-
break;
248-
}
253+
if (targetFile.exists()) {
254+
// It has already completed downloading.
255+
isInvalid = true;
256+
break;
257+
}
249258

250-
if (targetFile.exists()) {
251-
// It has already completed downloading.
252-
isInvalid = true;
253-
break;
254-
}
259+
} while (false);
260+
261+
262+
if (isInvalid) {
263+
maintainer.onRemovedInvalidData(model);
264+
removedDataCount.addAndGet(1);
265+
} else {
266+
final int oldId = model.getId();
267+
final int newId = idGenerator.transOldId(oldId, model.getUrl(),
268+
model.getPath(), model.isPathAsDirectory());
269+
if (newId != oldId) {
270+
if (FileDownloadLog.NEED_LOG) {
271+
FileDownloadLog.d(FileDownloadDatabase.class,
272+
"the id is changed on restoring from db:"
273+
+ " old[%d] -> new[%d]",
274+
oldId, newId);
275+
}
276+
model.setId(newId);
277+
maintainer.changeFileDownloadModelId(oldId, model);
278+
resetIdCount.addAndGet(1);
279+
}
255280

256-
} while (false);
257-
258-
259-
if (isInvalid) {
260-
iterator.remove();
261-
maintainer.onRemovedInvalidData(model);
262-
removedDataCount++;
263-
} else {
264-
final int oldId = model.getId();
265-
final int newId = idGenerator.transOldId(oldId, model.getUrl(), model.getPath(),
266-
model.isPathAsDirectory());
267-
if (newId != oldId) {
268-
if (FileDownloadLog.NEED_LOG) {
269-
FileDownloadLog.d(FileDownloadDatabase.class,
270-
"the id is changed on restoring from db:"
271-
+ " old[%d] -> new[%d]",
272-
oldId, newId);
281+
maintainer.onRefreshedValidData(model);
282+
refreshDataCount.addAndGet(1);
273283
}
274-
model.setId(newId);
275-
maintainer.changeFileDownloadModelId(oldId, model);
276-
resetIdCount++;
277284
}
278-
279-
maintainer.onRefreshedValidData(model);
280-
refreshDataCount++;
281-
}
285+
});
286+
futures.add(modelFuture);
282287
}
283288

284289
} finally {
285-
FileDownloadUtils.markConverted(FileDownloadHelper.getAppContext());
286-
maintainer.onFinishMaintain();
287-
// 566 data consumes about 140ms
288-
// update by rth: different devices have very large disparity, such as,
289-
// in my HuaWei Android 8.0, 67 data consumes about 355ms.
290-
// so, it's better do this action in background thread.
290+
final Future markConvertedFuture = maintainThreadPool.submit(new Runnable() {
291+
@Override
292+
public void run() {
293+
FileDownloadUtils.markConverted(FileDownloadHelper.getAppContext());
294+
}
295+
});
296+
futures.add(markConvertedFuture);
297+
final Future finishMaintainFuture = maintainThreadPool.submit(new Runnable() {
298+
@Override
299+
public void run() {
300+
maintainer.onFinishMaintain();
301+
}
302+
});
303+
futures.add(finishMaintainFuture);
304+
for (Future future : futures) {
305+
try {
306+
if (!future.isDone()) future.get();
307+
} catch (Exception e) {
308+
if (FileDownloadLog.NEED_LOG) FileDownloadLog
309+
.e(FileDownloadDatabase.class, e, e.getMessage());
310+
}
311+
}
312+
futures.clear();
291313
if (FileDownloadLog.NEED_LOG) {
292314
FileDownloadLog.d(FileDownloadDatabase.class,
293315
"refreshed data count: %d , delete data count: %d, reset id count:"
294316
+ " %d. consume %d",
295-
refreshDataCount, removedDataCount, resetIdCount,
317+
refreshDataCount.get(), removedDataCount.get(), resetIdCount.get(),
296318
System.currentTimeMillis() - startTimestamp);
297319
}
298320
}
299321
}
300-
}
322+
}

0 commit comments

Comments
 (0)