Skip to content

Commit 32a9a46

Browse files
qkaiserqkaiser
authored
Merge pull request #1163 from rxpha3l/erofs
feat(handler): add erofs filesystem & decompression handler
2 parents c505395 + 5f50cd2 commit 32a9a46

File tree

16 files changed

+237
-41
lines changed

16 files changed

+237
-41
lines changed

‎docs/formats.md

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@ For compression formats, metadata cannot be preserved, as this information in mo
9797
| ---------------------- | ---------------------------------- | ----------------------------------------------- | ----------------------------------------------- |
9898
| Android sparse image | ❌ | [filesystem/android/sparse.py][android-handler] | [`simg2img`][android-extractor] |
9999
| CRAMFS | ✅ | [filesystem/cramfs.py][cramfs-handler] | [`7z`][cramfs-extractor] |
100+
| EROFS | ✅ | [filesystem/android/erofs.py][erofs-handler] | [`fsck.erfos`][erofs-extractor] |
100101
| ExtFS | ✅ | [filesystem/extfs.py][extfs-handler] | [`debugfs`][extfs-extractor] |
101102
| FAT | ✅ | [filesystem/fat.py][fat-handler] | [`7z`][fat-extractor] |
102103
| ISO9660 | ✅ | [filesystem/iso9660.py][iso9660-handler] | [`7z`][iso9660-extractor] |
@@ -107,12 +108,14 @@ For compression formats, metadata cannot be preserved, as this information in mo
107108
| SquashFS v4 Big Endian | ✅ | [filesystem/squashfs.py][squashfs-handler] | [`sasquatch-v4-be`][squashfs-v4-be-extractor] |
108109
| UBI | ✅ | [filesystem/ubi.py][ubi-handler] | [`ubireader_extract_images`][ubi-extractor] |
109110
| UBIFS | ✅ | [filesystem/ubi.py][ubi-handler] | [`ubireader_extract_files`][ubifs-extractor] |
110-
| YAFFS (1, 2) | ✅ | [filesystem/yaffs.py][yaffs-handler] | [`YAFFSExtractor` custom code][yaffs-extractor] |
111+
| YAFFS (1, 2) | ✅ | [filesystem/yaffs.py][yaffs-handler] | [`YAFFSExtractor` custom code][yaffs-extractor] |
111112

112113
[android-handler]: https://github.com/onekey-sec/unblob/blob/main/unblob/handlers/filesystem/android/sparse.py
113114
[android-extractor]: https://github.com/onekey-sec/unblob/blob/3008039881a0434deb75962e7999b7e35aca8271/unblob/handlers/filesystem/android/sparse.py#L61
114115
[cramfs-handler]: https://github.com/onekey-sec/unblob/blob/main/unblob/handlers/filesystem/cramfs.py
115116
[cramfs-extractor]: https://github.com/onekey-sec/unblob/blob/3008039881a0434deb75962e7999b7e35aca8271/unblob/handlers/filesystem/cramfs.py#L45
117+
[erofs-handler]: https://github.com/onekey-sec/unblob/blob/main/unblob/handlers/filesystem/android/erfos.py
118+
[erofs-extractor]: https://github.com/onekey-sec/unblob/blob/main/unblob/handlers/filesystem/android/erfos.py#L45
116119
[extfs-handler]: https://github.com/onekey-sec/unblob/blob/main/unblob/handlers/filesystem/extfs.py
117120
[extfs-extractor]: https://github.com/onekey-sec/unblob/blob/3008039881a0434deb75962e7999b7e35aca8271/unblob/handlers/filesystem/extfs.py#L68
118121
[fat-handler]: https://github.com/onekey-sec/unblob/blob/main/unblob/handlers/filesystem/fat.py

‎install-deps.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ apt-get update
55
apt-get install --no-install-recommends -y \
66
android-sdk-libsparse-utils \
77
curl \
8+
erofs-utils \
89
lz4 \
910
lziprecover \
1011
lzop \

‎overlay.nix

Lines changed: 1 addition & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -8,42 +8,5 @@ final: prev:
88
nativeCheckInputs = (super.nativeCheckInputs or [ ]) ++ [ final.which ];
99
});
1010

11-
unblob =
12-
let
13-
pyproject_toml = (builtins.fromTOML (builtins.readFile ./pyproject.toml));
14-
version = pyproject_toml.project.version;
15-
in
16-
(prev.unblob.override { e2fsprogs = final.e2fsprogs-nofortify; }).overridePythonAttrs (super: rec {
17-
inherit version;
18-
19-
src = final.nix-filter {
20-
root = ./.;
21-
include = [
22-
"Cargo.lock"
23-
"Cargo.toml"
24-
"pyproject.toml"
25-
"python"
26-
"rust"
27-
"tests"
28-
"README.md"
29-
];
30-
};
31-
32-
dependencies = (super.dependencies or [ ]) ++ [ final.python3.pkgs.pyzstd ];
33-
34-
# remove this when packaging changes are upstreamed
35-
cargoDeps = final.rustPlatform.importCargoLock {
36-
lockFile = ./Cargo.lock;
37-
};
38-
39-
nativeBuildInputs = with final.rustPlatform; [
40-
cargoSetupHook
41-
maturinBuildHook
42-
];
43-
44-
# override disabling of 'test_all_handlers[filesystem.extfs]' from upstream
45-
pytestFlagsArray = [
46-
"--no-cov"
47-
];
48-
});
11+
unblob = final.callPackage ./package.nix { };
4912
}

‎package.nix

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
{
2+
lib,
3+
python3,
4+
fetchFromGitHub,
5+
makeWrapper,
6+
e2fsprogs-nofortify,
7+
erofs-utils,
8+
jefferson,
9+
lz4,
10+
lziprecover,
11+
lzop,
12+
p7zip,
13+
nix-filter,
14+
sasquatch,
15+
sasquatch-v4be,
16+
simg2img,
17+
ubi_reader,
18+
unar,
19+
zstd,
20+
versionCheckHook,
21+
rustPlatform,
22+
}:
23+
24+
let
25+
# These dependencies are only added to PATH
26+
runtimeDeps = [
27+
e2fsprogs-nofortify
28+
erofs-utils
29+
jefferson
30+
lziprecover
31+
lzop
32+
p7zip
33+
sasquatch
34+
sasquatch-v4be
35+
ubi_reader
36+
simg2img
37+
unar
38+
zstd
39+
lz4
40+
];
41+
pyproject_toml = (builtins.fromTOML (builtins.readFile ./pyproject.toml));
42+
version = pyproject_toml.project.version;
43+
in
44+
python3.pkgs.buildPythonApplication rec {
45+
pname = "unblob";
46+
pyproject = true;
47+
disabled = python3.pkgs.pythonOlder "3.9";
48+
inherit version;
49+
src = nix-filter {
50+
root = ./.;
51+
include = [
52+
"Cargo.lock"
53+
"Cargo.toml"
54+
"pyproject.toml"
55+
"python"
56+
"rust"
57+
"tests"
58+
"README.md"
59+
];
60+
};
61+
62+
strictDeps = true;
63+
64+
build-system = with python3.pkgs; [ poetry-core ];
65+
66+
dependencies = with python3.pkgs; [
67+
arpy
68+
attrs
69+
click
70+
cryptography
71+
dissect-cstruct
72+
lark
73+
lief.py
74+
python3.pkgs.lz4 # shadowed by pkgs.lz4
75+
plotext
76+
pluggy
77+
pyfatfs
78+
pyperscan
79+
python-magic
80+
pyzstd
81+
rarfile
82+
rich
83+
structlog
84+
treelib
85+
unblob-native
86+
];
87+
88+
cargoDeps = rustPlatform.importCargoLock {
89+
lockFile = ./Cargo.lock;
90+
};
91+
92+
nativeBuildInputs = with rustPlatform; [
93+
cargoSetupHook
94+
maturinBuildHook
95+
makeWrapper
96+
];
97+
98+
# These are runtime-only CLI dependencies, which are used through
99+
# their CLI interface
100+
pythonRemoveDeps = [
101+
"jefferson"
102+
"ubi-reader"
103+
];
104+
105+
pythonImportsCheck = [ "unblob" ];
106+
107+
makeWrapperArgs = [
108+
"--prefix PATH : ${lib.makeBinPath runtimeDeps}"
109+
];
110+
111+
nativeCheckInputs =
112+
with python3.pkgs;
113+
[
114+
pytestCheckHook
115+
pytest-cov
116+
versionCheckHook
117+
]
118+
++ runtimeDeps;
119+
120+
versionCheckProgramArg = "--version";
121+
122+
pytestFlagsArray = [
123+
"--no-cov"
124+
];
125+
126+
passthru = {
127+
# helpful to easily add these to a nix-shell environment
128+
inherit runtimeDeps;
129+
};
130+
131+
}

‎python/unblob/handlers/__init__.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@
4949
ubi,
5050
yaffs,
5151
)
52-
from .filesystem.android import sparse
52+
from .filesystem.android import erofs, sparse
5353

5454
BUILTIN_HANDLERS: Handlers = (
5555
cramfs.CramFSHandler,
@@ -118,6 +118,7 @@
118118
engenius.EngeniusHandler,
119119
ecc.AutelECCHandler,
120120
uzip.UZIPHandler,
121+
erofs.EROFSHandler,
121122
)
122123

123124
BUILTIN_DIR_HANDLERS: DirectoryHandlers = (

‎python/unblob/handlers/filesystem/android/erofs.py

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
import io
2+
from typing import Optional
3+
4+
from unblob.extractors import Command
5+
from unblob.file_utils import (
6+
Endian,
7+
InvalidInputFormat,
8+
snull,
9+
)
10+
from unblob.models import (
11+
File,
12+
HexString,
13+
StructHandler,
14+
ValidChunk,
15+
)
16+
17+
C_DEFINITIONS = r"""
18+
typedef struct erofs_handler{
19+
uint32_t magic;
20+
uint32_t crc32c;
21+
uint32_t feature_compact;
22+
uint8_t block_size_bs;
23+
uint8_t sb_extslots;
24+
uint16_t root_nid;
25+
uint64_t inos;
26+
uint64_t build_time;
27+
uint32_t build_time_nsec;
28+
uint32_t block_count;
29+
uint32_t meta_blkaddr;
30+
uint32_t xattr_blkaddr;
31+
uint8_t uuid[16];
32+
char volume_name[16];
33+
uint32_t feature_incompact;
34+
char reserved[44];
35+
} erofs_handler_t;
36+
"""
37+
38+
SUPERBLOCK_OFFSET = 0x400
39+
40+
41+
class EROFSHandler(StructHandler):
42+
NAME = "erofs"
43+
PATTERNS = [HexString("e2 e1 f5 e0")] # Magic in little endian
44+
HEADER_STRUCT = "erofs_handler_t"
45+
C_DEFINITIONS = C_DEFINITIONS
46+
EXTRACTOR = Command(
47+
"fsck.erofs",
48+
"--no-preserve",
49+
"--extract={outdir}",
50+
"{inpath}",
51+
)
52+
PATTERN_MATCH_OFFSET = -SUPERBLOCK_OFFSET
53+
54+
def is_valid_header(self, header) -> bool:
55+
try:
56+
snull(header.volume_name).decode("utf-8")
57+
except UnicodeDecodeError:
58+
return False
59+
return (
60+
header.block_count >= 1
61+
and header.build_time > 0
62+
and header.build_time_nsec > 0
63+
and header.block_size_bs >= 9
64+
)
65+
66+
def calculate_chunk(self, file: File, start_offset: int) -> Optional[ValidChunk]:
67+
file.seek(start_offset + SUPERBLOCK_OFFSET, io.SEEK_SET)
68+
header = self.parse_header(file, Endian.LITTLE)
69+
if not self.is_valid_header(header):
70+
raise InvalidInputFormat("Invalid erofs header.")
71+
72+
end_offset = (1 << header.block_size_bs) * header.block_count
73+
return ValidChunk(
74+
start_offset=start_offset,
75+
end_offset=end_offset,
76+
)

‎shell.nix

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ let
44
./flake.lock
55
./flake.nix
66
./overlay.nix
7-
./nix
7+
./package.nix
88
];
99

1010
lock = builtins.fromJSON (builtins.readFile ./flake.lock);

‎tests/integration/filesystem/android/erofs/__input__/foo.erofs.img

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
version https://git-lfs.github.com/spec/v1
2+
oid sha256:7f400d3497501c6e68324868d88207aa9398888902414045f4ea498ff26bcdcd
3+
size 4096

‎tests/integration/filesystem/android/erofs/__input__/foo_padding.erofs.img

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
version https://git-lfs.github.com/spec/v1
2+
oid sha256:7520d3fe98e5f5d25ccc4027321a561a125fbb413ecd83e935a8bc6b425b2aaf
3+
size 6144

‎tests/integration/filesystem/android/erofs/__output__/foo.erofs.img_extract/bar.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
version https://git-lfs.github.com/spec/v1
2+
oid sha256:0bfc3200f0152ab9e91e662afd75add3306131f670a1d71539f680c7acdb0a9f
3+
size 13

0 commit comments

Comments
 (0)