Skip to content

Commit 9f4ef4b

Browse files
author
Tianhua Ran
committed
refactor: speed up maintaining database
1 parent 4601b76 commit 9f4ef4b

File tree

5 files changed

+227
-149
lines changed

5 files changed

+227
-149
lines changed

‎demo/src/main/assets/filedownloader.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ http.lenient=false
99

1010
# The FileDownloadService runs in the separate process ':filedownloader' as default, if you want to
1111
# run the FileDownloadService in the main process, just set true. default false.
12-
process.non-separate=false
12+
process.non-separate=true
1313

1414
# The min buffered so far bytes.
1515
#

‎demo/src/main/java/com/liulishuo/filedownloader/demo/MainActivity.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ protected void onCreate(Bundle savedInstanceState) {
3131
// If you have such requirement, just implement FileDownloadMonitor.IMonitor, and register it
3232
// use FileDownloadDownloader.setGlobalMonitor the same as below code.
3333
FileDownloadMonitor.setGlobalMonitor(GlobalMonitor.getImpl());
34+
FileDownloader.getImpl().bindService();
3435
}
3536

3637
public void onClickMultitask(final View view) {

‎library/src/main/java/com/liulishuo/filedownloader/database/DatabaseMaintainer.java‎

Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
/*
2+
* Copyright (c) 2015 LingoChamp Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.liulishuo.filedownloader.database;
18+
19+
import com.liulishuo.filedownloader.model.FileDownloadModel;
20+
import com.liulishuo.filedownloader.model.FileDownloadStatus;
21+
import com.liulishuo.filedownloader.util.FileDownloadExecutors;
22+
import com.liulishuo.filedownloader.util.FileDownloadHelper;
23+
import com.liulishuo.filedownloader.util.FileDownloadLog;
24+
import com.liulishuo.filedownloader.util.FileDownloadUtils;
25+
26+
import java.io.File;
27+
import java.util.ArrayList;
28+
import java.util.Iterator;
29+
import java.util.List;
30+
import java.util.concurrent.Future;
31+
import java.util.concurrent.ThreadPoolExecutor;
32+
import java.util.concurrent.atomic.AtomicInteger;
33+
34+
public class DatabaseMaintainer {
35+
36+
private final ThreadPoolExecutor maintainThreadPool;
37+
private final FileDownloadDatabase.Maintainer maintainer;
38+
private final FileDownloadHelper.IdGenerator idGenerator;
39+
40+
public DatabaseMaintainer(FileDownloadDatabase.Maintainer maintainer, FileDownloadHelper.IdGenerator idGenerator) {
41+
this.maintainer = maintainer;
42+
this.idGenerator = idGenerator;
43+
this.maintainThreadPool = FileDownloadExecutors.newDefaultThreadPool(3,
44+
FileDownloadUtils.getThreadPoolName("MaintainDatabase"));
45+
}
46+
47+
public void doMaintainAction() {
48+
final Iterator<FileDownloadModel> iterator = maintainer.iterator();
49+
50+
final AtomicInteger removedDataCount = new AtomicInteger(0);
51+
final AtomicInteger resetIdCount = new AtomicInteger(0);
52+
final AtomicInteger refreshDataCount = new AtomicInteger(0);
53+
54+
final long startTimestamp = System.currentTimeMillis();
55+
final List<Future> futures = new ArrayList<>();
56+
int total = 0;
57+
try {
58+
while (iterator.hasNext()) {
59+
total++;
60+
final FileDownloadModel model = iterator.next();
61+
final Future modelFuture = maintainThreadPool.submit(new Runnable() {
62+
@Override
63+
public void run() {
64+
boolean isInvalid = false;
65+
do {
66+
if (model.getStatus() == FileDownloadStatus.progress
67+
|| model.getStatus() == FileDownloadStatus.connected
68+
|| model.getStatus() == FileDownloadStatus.error
69+
|| (model.getStatus() == FileDownloadStatus.pending && model
70+
.getSoFar() > 0)
71+
) {
72+
// Ensure can be covered by RESUME FROM BREAKPOINT.
73+
model.setStatus(FileDownloadStatus.paused);
74+
}
75+
final String targetFilePath = model.getTargetFilePath();
76+
if (targetFilePath == null) {
77+
// no target file path, can't used to resume from breakpoint.
78+
isInvalid = true;
79+
break;
80+
}
81+
82+
final File targetFile = new File(targetFilePath);
83+
// consider check in new thread, but SQLite lock | file lock aways effect, so
84+
// sync
85+
if (model.getStatus() == FileDownloadStatus.paused
86+
&& FileDownloadUtils.isBreakpointAvailable(model.getId(), model,
87+
model.getPath(), null)) {
88+
// can be reused in the old mechanism(no-temp-file).
89+
90+
final File tempFile = new File(model.getTempFilePath());
91+
92+
if (!tempFile.exists() && targetFile.exists()) {
93+
final boolean successRename = targetFile.renameTo(tempFile);
94+
if (FileDownloadLog.NEED_LOG) {
95+
FileDownloadLog.d(DatabaseMaintainer.class,
96+
"resume from the old no-temp-file architecture "
97+
+ "[%B], [%s]->[%s]",
98+
successRename, targetFile.getPath(), tempFile.getPath());
99+
100+
}
101+
}
102+
}
103+
104+
/**
105+
* Remove {@code model} from DB if it can't used for judging whether the
106+
* old-downloaded file is valid for reused & it can't used for resuming from
107+
* BREAKPOINT, In other words, {@code model} is no use anymore for
108+
* FileDownloader.
109+
*/
110+
if (model.getStatus() == FileDownloadStatus.pending && model.getSoFar() <= 0) {
111+
// This model is redundant.
112+
isInvalid = true;
113+
break;
114+
}
115+
116+
if (!FileDownloadUtils.isBreakpointAvailable(model.getId(), model)) {
117+
// It can't used to resuming from breakpoint.
118+
isInvalid = true;
119+
break;
120+
}
121+
122+
if (targetFile.exists()) {
123+
// It has already completed downloading.
124+
isInvalid = true;
125+
break;
126+
}
127+
128+
} while (false);
129+
130+
if (isInvalid) {
131+
maintainer.onRemovedInvalidData(model);
132+
removedDataCount.addAndGet(1);
133+
} else {
134+
final int oldId = model.getId();
135+
final int newId = idGenerator.transOldId(oldId,
136+
model.getUrl(), model.getPath(),
137+
model.isPathAsDirectory());
138+
if (newId != oldId) {
139+
if (FileDownloadLog.NEED_LOG) {
140+
FileDownloadLog.d(DatabaseMaintainer.class,
141+
"the id is changed on restoring from db:"
142+
+ " old[%d] -> new[%d]",
143+
oldId, newId);
144+
}
145+
model.setId(newId);
146+
maintainer.changeFileDownloadModelId(oldId, model);
147+
resetIdCount.addAndGet(1);
148+
}
149+
150+
maintainer.onRefreshedValidData(model);
151+
refreshDataCount.addAndGet(1);
152+
}
153+
}
154+
});
155+
futures.add(modelFuture);
156+
}
157+
} finally {
158+
final Future markConvertedFuture = maintainThreadPool.submit(new Runnable() {
159+
@Override
160+
public void run() {
161+
FileDownloadUtils.markConverted(FileDownloadHelper.getAppContext());
162+
}
163+
});
164+
futures.add(markConvertedFuture);
165+
final Future finishMaintainFuture = maintainThreadPool.submit(new Runnable() {
166+
@Override
167+
public void run() {
168+
maintainer.onFinishMaintain();
169+
}
170+
});
171+
futures.add(finishMaintainFuture);
172+
for (Future future : futures) {
173+
try {
174+
if (!future.isDone()) future.get();
175+
} catch (Exception e) {
176+
if (FileDownloadLog.NEED_LOG) FileDownloadLog.e(this, e, e.getMessage());
177+
}
178+
}
179+
futures.clear();
180+
if (FileDownloadLog.NEED_LOG) {
181+
FileDownloadLog.d(this,
182+
"refreshed data count: %d , delete data count: %d, reset id count:"
183+
+ " %d. consume %d, total: %d",
184+
refreshDataCount.get(), removedDataCount.get(), resetIdCount.get(),
185+
System.currentTimeMillis() - startTimestamp, total);
186+
}
187+
}
188+
}
189+
}

‎library/src/main/java/com/liulishuo/filedownloader/database/SqliteDatabaseImpl.java‎

Lines changed: 33 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -35,22 +35,21 @@
3535

3636
/**
3737
* Persist data to SQLite database.
38-
*
38+
* <p>
3939
* You can valid this database implementation through:
4040
* <p>
4141
* class MyApplication extends Application {
42-
* ...
43-
* public void onCreate() {
44-
* ...
45-
* FileDownloader.setupOnApplicationOnCreate(this)
46-
* .database(SqliteDatabaseImpl.createMaker())
47-
* ...
48-
* .commit();
49-
* ...
50-
* }
51-
* ...
42+
* ...
43+
* public void onCreate() {
44+
* ...
45+
* FileDownloader.setupOnApplicationOnCreate(this)
46+
* .database(SqliteDatabaseImpl.createMaker())
47+
* ...
48+
* .commit();
49+
* ...
50+
* }
51+
* ...
5252
* }
53-
*
5453
*/
5554
public class SqliteDatabaseImpl implements FileDownloadDatabase {
5655

@@ -70,7 +69,8 @@ public SqliteDatabaseImpl() {
7069
db = openHelper.getWritableDatabase();
7170
}
7271

73-
@Override public void onTaskStart(int id) {
72+
@Override
73+
public void onTaskStart(int id) {
7474
}
7575

7676
@Override
@@ -267,6 +267,7 @@ private void update(final int id, final ContentValues cv) {
267267
public class Maintainer implements FileDownloadDatabase.Maintainer {
268268

269269
private final SparseArray<FileDownloadModel> needChangeIdList = new SparseArray<>();
270+
private final SparseArray<FileDownloadModel> needRemoveList = new SparseArray<>();
270271
private MaintainerIterator currentIterator;
271272

272273
private final SparseArray<FileDownloadModel> downloaderModelMap;
@@ -317,6 +318,14 @@ public void onFinishMaintain() {
317318
}
318319
}
319320

321+
// remove invalid
322+
final int removeSize = needRemoveList.size();
323+
for (int i = 0; i < removeSize; i++) {
324+
final int modelId = needRemoveList.keyAt(i);
325+
db.delete(TABLE_NAME, FileDownloadModel.ID + " = ?",
326+
new String[]{String.valueOf(modelId)});
327+
}
328+
320329
// initial cache of connection model
321330
if (downloaderModelMap != null && connectionModelListMap != null) {
322331
final int size = downloaderModelMap.size();
@@ -339,16 +348,25 @@ public void onFinishMaintain() {
339348

340349
@Override
341350
public void onRemovedInvalidData(FileDownloadModel model) {
351+
synchronized (needRemoveList) {
352+
needRemoveList.put(model.getId(), model);
353+
}
342354
}
343355

344356
@Override
345357
public void onRefreshedValidData(FileDownloadModel model) {
346-
if (downloaderModelMap != null) downloaderModelMap.put(model.getId(), model);
358+
if (downloaderModelMap != null) {
359+
synchronized (downloaderModelMap) {
360+
downloaderModelMap.put(model.getId(), model);
361+
}
362+
}
347363
}
348364

349365
@Override
350366
public void changeFileDownloadModelId(int oldId, FileDownloadModel modelWithNewId) {
351-
needChangeIdList.put(oldId, modelWithNewId);
367+
synchronized (needChangeIdList) {
368+
needChangeIdList.put(oldId, modelWithNewId);
369+
}
352370
}
353371

354372
}

0 commit comments

Comments
 (0)