Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
65 commits
Select commit Hold shift + click to select a range
18dad5e
feat(test): add exact coverage tests for fingerprint functionality
3-Tokisaki-Kurumi Jul 12, 2025
ae30976
feat(test): add comprehensive fingerprint core functionality tests
3-Tokisaki-Kurumi Jul 12, 2025
80d4efb
feat(test): add fingerprint coverage-specific tests for 100% code cov…
3-Tokisaki-Kurumi Jul 12, 2025
513b406
feat(test): add fingerprint spoofing integration tests
3-Tokisaki-Kurumi Jul 12, 2025
969649c
feat(test): add fingerprint refactoring verification tests
3-Tokisaki-Kurumi Jul 12, 2025
481c168
feat(examples): add comprehensive fingerprint spoofing example
3-Tokisaki-Kurumi Jul 12, 2025
841f1d8
feat(constants): add comprehensive fingerprint spoofing JavaScript co…
3-Tokisaki-Kurumi Jul 12, 2025
572d5d7
feat(fingerprint): add fingerprint module initialization and exports
3-Tokisaki-Kurumi Jul 12, 2025
cca5812
feat(fingerprint): add sophisticated fingerprint generation engine
3-Tokisaki-Kurumi Jul 12, 2025
c7ff166
feat(fingerprint): add fingerprint module initialization and exports
3-Tokisaki-Kurumi Jul 12, 2025
5e30ad9
feat(fingerprint): add fingerprint lifecycle management system
3-Tokisaki-Kurumi Jul 12, 2025
b1996b4
feat(fingerprint): add fingerprint data models and configuration classes
3-Tokisaki-Kurumi Jul 12, 2025
39d97c9
feat(browser): add fingerprint spoofing integration to browser base c…
3-Tokisaki-Kurumi Jul 12, 2025
3a2bc4d
feat(browser): add fingerprint spoofing integration to browser base c…
3-Tokisaki-Kurumi Jul 12, 2025
6529c60
feat(browser): add fingerprint spoofing parameters to Chrome browser
3-Tokisaki-Kurumi Jul 12, 2025
55e99d7
feat(browser): add fingerprint spoofing parameters to Edge browser
3-Tokisaki-Kurumi Jul 12, 2025
3d8c052
refactor(elements): improve option element clicking implementation
3-Tokisaki-Kurumi Jul 12, 2025
13565ff
refactor(protocol/browser): standardize typing imports in parameters
3-Tokisaki-Kurumi Jul 12, 2025
a7e2b22
refactor(protocol/browser): standardize typing imports in type defini…
3-Tokisaki-Kurumi Jul 12, 2025
35b2440
refactor(protocol/dom): standardize typing imports and remove unused …
3-Tokisaki-Kurumi Jul 12, 2025
0df2352
refactor(protocol/dom): standardize typing imports in response defini…
3-Tokisaki-Kurumi Jul 12, 2025
031a213
refactor(protocol/dom): standardize typing imports in type definitions
3-Tokisaki-Kurumi Jul 12, 2025
01d8158
refactor(protocol/fetch): standardize typing imports in parameters
3-Tokisaki-Kurumi Jul 12, 2025
eeb3532
refactor(protocol/fetch): standardize typing imports in type definitions
3-Tokisaki-Kurumi Jul 12, 2025
cfabd16
refactor(protocol): modernize base protocol types and simplify comple…
3-Tokisaki-Kurumi Jul 12, 2025
312e10b
refactor(protocol/input): standardize typing imports in input parameters
3-Tokisaki-Kurumi Jul 12, 2025
d7bf446
refactor(protocol/input): standardize typing imports in input type de…
3-Tokisaki-Kurumi Jul 12, 2025
d217c09
refactor(protocol/network): standardize typing imports in network par…
3-Tokisaki-Kurumi Jul 12, 2025
d7652e9
refactor(protocol/network): standardize typing imports in network res…
3-Tokisaki-Kurumi Jul 12, 2025
55777a8
refactor(protocol/network): standardize typing imports in network typ…
3-Tokisaki-Kurumi Jul 12, 2025
3ad67f5
refactor(protocol/page): standardize typing imports and enhance searc…
3-Tokisaki-Kurumi Jul 12, 2025
8f72c98
refactor(protocol/page): standardize typing imports in page responses
3-Tokisaki-Kurumi Jul 12, 2025
4acf614
refactor(protocol/page): standardize typing imports and complete Visu…
3-Tokisaki-Kurumi Jul 12, 2025
1506b95
refactor(protocol/runtime): standardize typing imports in runtime par…
3-Tokisaki-Kurumi Jul 12, 2025
cf382f2
refactor(protocol/runtime): standardize typing imports in runtime res…
3-Tokisaki-Kurumi Jul 12, 2025
292f306
refactor(protocol/runtime): standardize typing imports in runtime types
3-Tokisaki-Kurumi Jul 12, 2025
f99f87f
refactor(protocol/storage): standardize typing imports in storage par…
3-Tokisaki-Kurumi Jul 12, 2025
f99ae49
refactor(protocol/storage): standardize typing imports in storage types
3-Tokisaki-Kurumi Jul 12, 2025
fdce644
refactor(protocol/target): standardize typing imports in target param…
3-Tokisaki-Kurumi Jul 12, 2025
0a74fa8
refactor(protocol/target): standardize typing imports in target types
3-Tokisaki-Kurumi Jul 12, 2025
ae3e238
fix(fingerprint): improve randomness in fingerprint generation
3-Tokisaki-Kurumi Jul 12, 2025
29089cb
Merge branch 'autoscrape-labs:main' into main
3-Tokisaki-Kurumi Jul 13, 2025
8e22f99
Merge branch 'main' into main
3-Tokisaki-Kurumi Jul 14, 2025
3a6066a
fix: A bug fix
3-Tokisaki-Kurumi Jul 14, 2025
76d7c26
refactor: Optimize import
3-Tokisaki-Kurumi Jul 14, 2025
18a50f6
feat:Refactoring code to abstract repetitive logic into a method
3-Tokisaki-Kurumi Jul 14, 2025
60f538c
refactor: Add appropriate logging records
3-Tokisaki-Kurumi Jul 14, 2025
e4b032f
style: Simplify if else statements into ternary operators
3-Tokisaki-Kurumi Jul 14, 2025
36df6ce
feat(examples): Add a new example,update fingerprint config with vali…
3-Tokisaki-Kurumi Jul 14, 2025
2f37537
refactor(browser): remove duplicate fingerprint parameters from Chrom…
3-Tokisaki-Kurumi Jul 14, 2025
111a90a
refactor(browser): remove duplicate fingerprint parameters from Edge …
3-Tokisaki-Kurumi Jul 14, 2025
54cd3d9
fix(options): add automatic dict to FingerprintConfig conversion
3-Tokisaki-Kurumi Jul 14, 2025
1f781c1
feat(options): add fingerprint configuration properties to ChromiumOp…
3-Tokisaki-Kurumi Jul 14, 2025
8bce279
style:Fixed string formatting issue
3-Tokisaki-Kurumi Jul 14, 2025
7abd8d9
fix:Fix check logic that is too broad
3-Tokisaki-Kurumi Jul 14, 2025
50d9cbe
refactor:Update API
3-Tokisaki-Kurumi Jul 14, 2025
e493bdc
refactor:Use new API
3-Tokisaki-Kurumi Jul 14, 2025
fe78faa
refactor:Update API
3-Tokisaki-Kurumi Jul 14, 2025
e855c81
refactor:Update API
3-Tokisaki-Kurumi Jul 14, 2025
beb19d7
style:All checks passed!
3-Tokisaki-Kurumi Jul 14, 2025
5c19309
fix:Fix mypy type check errors
3-Tokisaki-Kurumi Jul 14, 2025
ee6c9d4
fix:Fix the mismatch between testing and implementation
3-Tokisaki-Kurumi Jul 14, 2025
3c7a912
Merge branch 'autoscrape-labs:main' into main
3-Tokisaki-Kurumi Jul 17, 2025
199315d
Merge branch 'main' into main
3-Tokisaki-Kurumi Jul 26, 2025
34169f9
Revert "Merge branch 'main' into main"
3-Tokisaki-Kurumi Jul 26, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
420 changes: 420 additions & 0 deletions examples/fingerprint_example.py

Large diffs are not rendered by default.

133 changes: 133 additions & 0 deletions examples/refactored_api_example.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
#!/usr/bin/env python3
"""
Example demonstrating the refactored fingerprint options API.

This example shows how fingerprint-related options are now centralized
in the ChromiumOptions class, making the API cleaner and more maintainable.
"""

import asyncio

from pydoll.browser.chromium.chrome import Chrome
from pydoll.browser.chromium.edge import Edge
from pydoll.browser.options import ChromiumOptions


async def demonstrate_new_api():
"""Demonstrate the new centralized options API."""

print("=== Refactored API Demo: Centralized Fingerprint Options ===\n")

# Example 1: Basic Chrome usage without fingerprint spoofing
print("1. Basic Chrome usage (no fingerprint spoofing):")
options_basic = ChromiumOptions()
options_basic.add_argument('--window-size=1024,768')

async with Chrome(options=options_basic) as chrome:
print(f" Fingerprint spoofing: {chrome.enable_fingerprint_spoofing}")
print(" ✓ Basic Chrome instance created successfully\n")

# Example 2: Enable fingerprint spoofing with the convenience method
print("2. Chrome with fingerprint spoofing (convenience method):")
options_spoofing = ChromiumOptions()
options_spoofing.enable_fingerprint_spoofing_mode() # Enable with default config

async with Chrome(options=options_spoofing) as chrome:
print(f" Fingerprint spoofing: {chrome.enable_fingerprint_spoofing}")
print(" ✓ Fingerprint spoofing enabled successfully\n")

# Example 3: Enable fingerprint spoofing with custom configuration
print("3. Chrome with custom fingerprint configuration:")
options_custom = ChromiumOptions()
custom_config = {
"browser_type": "chrome",
"is_mobile": False,
"preferred_os": "windows",
"preferred_languages": ["zh-CN", "en-US"],
"enable_webgl_spoofing": True,
"enable_canvas_spoofing": True
}
options_custom.enable_fingerprint_spoofing_mode(config=custom_config)

async with Chrome(options=options_custom) as chrome:
print(f" Fingerprint spoofing: {chrome.enable_fingerprint_spoofing}")
print(f" Custom config applied: {options_custom.fingerprint_config}")
print(" ✓ Custom fingerprint configuration applied\n")

# Example 4: Using property setters for fine-grained control
print("4. Using property setters for fine-grained control:")
options_properties = ChromiumOptions()
options_properties.enable_fingerprint_spoofing = True
options_properties.fingerprint_config = {
"enable_webgl_spoofing": False,
"include_plugins": False
}
options_properties.add_argument('--disable-blink-features=AutomationControlled')

async with Chrome(options=options_properties) as chrome:
print(f" Fingerprint spoofing: {chrome.enable_fingerprint_spoofing}")
print(f" Config via property: {options_properties.fingerprint_config}")
print(" ✓ Property setters work perfectly\n")

# Example 5: Same pattern works for Edge
print("5. Edge browser with fingerprint spoofing:")
options_edge = ChromiumOptions()
options_edge.enable_fingerprint_spoofing_mode()

async with Edge(options=options_edge) as edge:
print(f" Edge fingerprint spoofing: {edge.enable_fingerprint_spoofing}")
print(" ✓ Edge browser with fingerprint spoofing enabled\n")

print("=== Key Benefits of the Refactored API ===")
print("✓ Centralized configuration: All fingerprint settings in options")
print("✓ Cleaner constructors: No need for extra fingerprint parameters")
print("✓ Better maintainability: One place to manage fingerprint options")
print("✓ Consistent API: Same pattern for Chrome and Edge")
print("✓ Flexibility: Multiple ways to configure (convenience method + properties)")


def demonstrate_usage_patterns():
"""Show different usage patterns for the new API."""

print("\n=== Different Usage Patterns ===\n")

# Pattern 1: Method chaining style
print("Pattern 1: Method chaining style")
options1 = ChromiumOptions()
options1.enable_fingerprint_spoofing_mode({"method": "chaining"})
options1.add_argument('--no-sandbox')
print(" ✓ Configuration complete\n")

# Pattern 2: Property assignment style
print("Pattern 2: Property assignment style")
options2 = ChromiumOptions()
options2.enable_fingerprint_spoofing = True
options2.fingerprint_config = {"browser_type": "chrome", "is_mobile": False}
print(" ✓ Configuration complete\n")

# Pattern 3: Build pattern
print("Pattern 3: Builder pattern")

def build_options_with_fingerprint(config=None):
options = ChromiumOptions()
options.enable_fingerprint_spoofing_mode(config)
options.add_argument('--disable-web-security')
return options

_options3 = build_options_with_fingerprint({"builder": "pattern"})
print(" ✓ Builder pattern complete\n")

print("All patterns provide the same clean, centralized configuration!")


if __name__ == "__main__":
print("Running refactored API demonstration...")

# Demonstrate the new API patterns
demonstrate_usage_patterns()

# Run the async demo
asyncio.run(demonstrate_new_api())

print("\n🎉 Refactored API demonstration complete!")
print("The fingerprint options are now properly centralized in ChromiumOptions!")
100 changes: 92 additions & 8 deletions pydoll/browser/chromium/base.py
Original file line number Diff line number Diff line change
@@ -1,19 +1,23 @@
import asyncio
import logging
from abc import ABC, abstractmethod
from functools import partial
from random import randint
from typing import Any, Callable, Optional, TypeVar
from typing import TYPE_CHECKING, Any, Callable, Optional, TypeVar

from pydoll.browser.interfaces import BrowserOptionsManager
from pydoll.browser.managers import (
BrowserProcessManager,
ProxyManager,
TempDirectoryManager,
)
from pydoll.browser.tab import Tab

if TYPE_CHECKING:
from pydoll.browser.tab import Tab
from pydoll.commands import (
BrowserCommands,
FetchCommands,
PageCommands,
RuntimeCommands,
StorageCommands,
TargetCommands,
Expand Down Expand Up @@ -49,6 +53,8 @@

T = TypeVar('T')

logger = logging.getLogger(__name__)


class Browser(ABC): # noqa: PLR0904
"""
Expand Down Expand Up @@ -82,6 +88,12 @@ def __init__(
self._temp_directory_manager = TempDirectoryManager()
self._connection_handler = ConnectionHandler(self._connection_port)

# Store fingerprint manager reference if available
self.fingerprint_manager = getattr(options_manager, 'fingerprint_manager', None)
self.enable_fingerprint_spoofing = getattr(
options_manager, 'enable_fingerprint_spoofing', False
)

async def __aenter__(self) -> 'Browser':
"""Async context manager entry."""
return self
Expand All @@ -93,7 +105,7 @@ async def __aexit__(self, exc_type, exc_val, exc_tb):

await self._connection_handler.close()

async def start(self, headless: bool = False) -> Tab:
async def start(self, headless: bool = False) -> 'Tab':
"""
Start browser process and establish CDP connection.

Expand Down Expand Up @@ -124,8 +136,16 @@ async def start(self, headless: bool = False) -> Tab:
await self._verify_browser_running()
await self._configure_proxy(proxy_config[0], proxy_config[1])

# Import at runtime to avoid circular import
from pydoll.browser.tab import Tab # noqa: PLC0415

valid_tab_id = await self._get_valid_tab_id(await self.get_targets())
return Tab(self, self._connection_port, valid_tab_id)
tab = Tab(self, self._connection_port, valid_tab_id)

# Inject fingerprint spoofing JavaScript if enabled
await self._setup_fingerprint_for_tab(tab)

return tab

async def stop(self):
"""
Expand Down Expand Up @@ -190,7 +210,7 @@ async def get_browser_contexts(self) -> list[str]:
)
return response['result']['browserContextIds']

async def new_tab(self, url: str = '', browser_context_id: Optional[str] = None) -> Tab:
async def new_tab(self, url: str = '', browser_context_id: Optional[str] = None) -> 'Tab':
"""
Create new tab for page interaction.

Expand All @@ -203,12 +223,20 @@ async def new_tab(self, url: str = '', browser_context_id: Optional[str] = None)
"""
response: CreateTargetResponse = await self._execute_command(
TargetCommands.create_target(
url=url,
browser_context_id=browser_context_id,
)
)
target_id = response['result']['targetId']

# Import at runtime to avoid circular import
from pydoll.browser.tab import Tab # noqa: PLC0415

tab = Tab(self, self._connection_port, target_id, browser_context_id)
if url: await tab.go_to(url)

# Inject fingerprint spoofing JavaScript if enabled
await self._setup_fingerprint_for_tab(tab)

return tab

async def get_targets(self) -> list[TargetInfo]:
Expand All @@ -224,7 +252,7 @@ async def get_targets(self) -> list[TargetInfo]:
response: GetTargetsResponse = await self._execute_command(TargetCommands.get_targets())
return response['result']['targetInfos']

async def get_opened_tabs(self) -> list[Tab]:
async def get_opened_tabs(self) -> list['Tab']:
"""
Get all opened tabs that are not extensions and have the type 'page'

Expand All @@ -237,6 +265,10 @@ async def get_opened_tabs(self) -> list[Tab]:
for target in targets
if target['type'] == 'page' and 'extension' not in target['url']
]

# Import at runtime to avoid circular import
from pydoll.browser.tab import Tab # noqa: PLC0415

return [
Tab(self, self._connection_port, target['targetId'])
for target in reversed(valid_tab_targets)
Expand Down Expand Up @@ -299,7 +331,7 @@ async def get_window_id_for_target(self, target_id: str) -> int:
)
return response['result']['windowId']

async def get_window_id_for_tab(self, tab: Tab) -> int:
async def get_window_id_for_tab(self, tab: 'Tab') -> int:
"""Get window ID for tab (convenience method)."""
return await self.get_window_id_for_target(tab._target_id)

Expand Down Expand Up @@ -585,6 +617,58 @@ def _setup_user_dir(self):
temp_dir = self._temp_directory_manager.create_temp_dir()
self.options.arguments.append(f'--user-data-dir={temp_dir.name}')

async def _setup_fingerprint_for_tab(self, tab):
"""
Setup fingerprint spoofing for a tab if enabled.

Args:
tab: The tab to setup fingerprint spoofing for.
"""
if self.enable_fingerprint_spoofing and self.fingerprint_manager:
await self._inject_fingerprint_script(tab)

async def _inject_fingerprint_script(self, tab):
"""
Inject fingerprint spoofing JavaScript into a tab.

Args:
tab: The tab to inject the script into.
"""
try:
# Get the JavaScript injection code
assert self.fingerprint_manager is not None
script = self.fingerprint_manager.get_fingerprint_js()

# Inject the script using Page.addScriptToEvaluateOnNewDocument
# This ensures the script runs before any page scripts
await tab._execute_command(
PageCommands.add_script_to_evaluate_on_new_document(script)
)

# Also evaluate immediately for current page if it exists
try:
await tab.execute_script(script)
except (RuntimeError, OSError, ValueError):
# Ignore errors for immediate execution as page might not be ready
pass

except (RuntimeError, OSError, ValueError, AssertionError) as e:
# Don't let fingerprint injection failures break the browser
logger.warning("Failed to inject fingerprint spoofing script: %s", e)

def get_fingerprint_summary(self) -> Optional[dict]:
"""
Get a summary of the current fingerprint.

Returns:
Dictionary with fingerprint information, or None if not enabled.
"""
return (
self.fingerprint_manager.get_fingerprint_summary()
if self.fingerprint_manager
else None
)

@abstractmethod
def _get_default_binary_location(self) -> str:
"""Get default browser executable path (implemented by subclasses)."""
Expand Down
7 changes: 6 additions & 1 deletion pydoll/browser/chromium/chrome.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import platform
from typing import Optional
from typing import Optional, cast

from pydoll.browser.chromium.base import Browser
from pydoll.browser.managers import ChromiumOptionsManager
Expand All @@ -25,6 +25,11 @@ def __init__(
"""
options_manager = ChromiumOptionsManager(options)
super().__init__(options_manager, connection_port)
# Get fingerprint settings from already initialized options
# Cast to ChromiumOptions to access fingerprint properties
chromium_options = cast(ChromiumOptions, self.options)
self.enable_fingerprint_spoofing = chromium_options.enable_fingerprint_spoofing
self.fingerprint_manager = options_manager.get_fingerprint_manager()

@staticmethod
def _get_default_binary_location():
Expand Down
11 changes: 8 additions & 3 deletions pydoll/browser/chromium/edge.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import platform
from typing import Optional
from typing import Optional, cast

from pydoll.browser.chromium.base import Browser
from pydoll.browser.managers import ChromiumOptionsManager
from pydoll.browser.options import Options
from pydoll.browser.options import ChromiumOptions
from pydoll.exceptions import UnsupportedOS
from pydoll.utils import validate_browser_paths

Expand All @@ -13,7 +13,7 @@ class Edge(Browser):

def __init__(
self,
options: Optional[Options] = None,
options: Optional[ChromiumOptions] = None,
connection_port: Optional[int] = None,
):
"""
Expand All @@ -25,6 +25,11 @@ def __init__(
"""
options_manager = ChromiumOptionsManager(options)
super().__init__(options_manager, connection_port)
# Get fingerprint settings from already initialized options
# Cast to ChromiumOptions to access fingerprint properties
chromium_options = cast(ChromiumOptions, self.options)
self.enable_fingerprint_spoofing = chromium_options.enable_fingerprint_spoofing
self.fingerprint_manager = options_manager.get_fingerprint_manager()

@staticmethod
def _get_default_binary_location():
Expand Down
Loading
Loading