Provide option for disabling reading from already finished files

Author: Roman513

README.md

@@ -35,7 +35,7 @@ it has to be run from a terminal (e.g. PowerShell)
 - Linux, Windows (8.1+), or macOS (12.0+)
   + 32-bit operating systems are not supported
 - python 3.9+ (64-bit)
-  + (Windows) `pythonnet` is not yet compatible with 3.10+, use 3.9 if you plan to install `pywebview` 
+  + (Windows) `pythonnet` is not yet compatible with 3.10+, use 3.9 if you plan to install `pywebview`
 - PyPI packages:
   + `requests`
   + (optional) `pywebview` for webview-based login
@@ -71,7 +71,7 @@ Note that on Linux glibc >= 2.25 is required, so older distributions such as Ubu
 
 #### Prerequisites
 
-To prevent problems with permissions during installation, please upgrade your `pip` by running `python -m pip install -U pip --user`. 
+To prevent problems with permissions during installation, please upgrade your `pip` by running `python -m pip install -U pip --user`.
 
 > **Tip:** You may need to replace `python` in the above command with `python3` on Linux/macOS, or `py -3` on Windows.
 
@@ -87,7 +87,7 @@ Optionally if logging in via an embedded web view is desired also run
 ```bash
 pip install legendary-gl[webview]
 ```
-On Linux this may also require installing a supported web engine and its python bindings.  
+On Linux this may also require installing a supported web engine and its python bindings.
 Ubunutu example:
 ```bash
 sudo apt install python3-gi-cairo
@@ -115,7 +115,7 @@ cd legendary
 pip install .
 ````
 
-If the `legendary` executable is not available after installation, you may need to configure your `PATH` correctly. You can do this by running the command: 
+If the `legendary` executable is not available after installation, you may need to configure your `PATH` correctly. You can do this by running the command:
 
 ```bash
 echo 'export PATH=$PATH:~/.local/bin' >> ~/.profile && source ~/.profile
@@ -174,7 +174,7 @@ Importing a previously installed game
 ````
 legendary import Anemone /mnt/games/Epic/WorldOfGoo
 ````
-**Note:** Importing will require a full verification so Legendary can correctly update the game later.  
+**Note:** Importing will require a full verification so Legendary can correctly update the game later.
 **Note 2:** In order to use an alias here you may have to put it into quotes if if contains more than one word, e.g. `legendary import-game "world of goo" /mnt/games/Epic/WorldOfGoo`.
 
 Sync savegames with the Epic Cloud
@@ -445,6 +445,8 @@ optional arguments:
   --force               Download all files / ignore existing (overwrite)
   --disable-patching    Do not attempt to patch existing installation
                         (download entire changed files)
+  --disable-read-files  Do not re-read duplicated parts from already saved
+                        files, keep them in memory
   --download-only, --no-install
                         Do not install app and do not run prerequisite
                         installers after download
@@ -677,7 +679,7 @@ locale = en-US
 ; whether or not syncing with egl is enabled
 egl_sync = false
 ; path to the "Manifests" folder in the EGL ProgramData directory
-egl_programdata = /home/user/Games/epic-games-store/drive_c/... 
+egl_programdata = /home/user/Games/epic-games-store/drive_c/...
 ; Set preferred CDN host (e.g. to improve download speed)
 preferred_cdn = epicgames-download1.akamaized.net
 ; disable HTTPS for downloads (e.g. to use a LanCache)

legendary/cli.py

@@ -964,6 +964,7 @@ def install_game(self, args):
                                                           force=args.force, max_shm=args.shared_memory,
                                                           max_workers=args.max_workers, game_folder=args.game_folder,
                                                           disable_patching=args.disable_patching,
+                                                          disable_read_files=args.disable_read_files,
                                                           override_manifest=args.override_manifest,
                                                           override_old_manifest=args.override_old_manifest,
                                                           override_base_url=args.override_base_url,
@@ -1197,7 +1198,7 @@ def _handle_uninstaller(self, igame, yes=False):
 
         print('\nThis game provides the following uninstaller:')
         print(f'- {uninstaller["path"]} {uninstaller["args"]}\n')
-        
+
         if yes or get_boolean_choice('Do you wish to run the uninstaller?', default=True):
             logger.info('Running uninstaller...')
             req_path, req_exec = os.path.split(uninstaller['path'])
@@ -2470,7 +2471,7 @@ def crossover_setup(self, args):
                 default_choice = None
                 for i, bottle in enumerate(usable_bottles, start=1):
                     extra = []
-                    
+
                     if cx_version in bottle['cx_versions']:
                         if app_name in bottle['compatible_apps']:
                             extra.append('recommended')
@@ -2754,6 +2755,8 @@ def main():
                                 help='Download all files / ignore existing (overwrite)')
     install_parser.add_argument('--disable-patching', dest='disable_patching', action='store_true',
                                 help='Do not attempt to patch existing installation (download entire changed files)')
+    install_parser.add_argument('--disable-read-files', dest='disable_read_files', action='store_true',
+                                help='Do not re-read duplicated parts from already saved files, keep them in memory')
     install_parser.add_argument('--download-only', '--no-install', dest='no_install', action='store_true',
                                 help='Do not install app and do not run prerequisite installers after download')
     install_parser.add_argument('--update-only', dest='update_only', action='store_true',

legendary/core.py

@@ -1323,6 +1323,7 @@ def get_delta_manifest(self, base_url, old_build_id, new_build_id):
     def prepare_download(self, game: Game, base_game: Game = None, base_path: str = '',
                          status_q: Queue = None, max_shm: int = 0, max_workers: int = 0,
                          force: bool = False, disable_patching: bool = False,
+                         disable_read_files: bool = False,
                          game_folder: str = '', override_manifest: str = '',
                          override_old_manifest: str = '', override_base_url: str = '',
                          platform: str = 'Windows', file_prefix_filter: list = None,
@@ -1501,6 +1502,7 @@ def prepare_download(self, game: Game, base_game: Game = None, base_path: str =
                         dl_timeout=dl_timeout, bind_ip=bind_ip)
         anlres = dlm.run_analysis(manifest=new_manifest, old_manifest=old_manifest,
                                   patch=not disable_patching, resume=not force,
+                                  read_files=not disable_read_files,
                                   file_prefix_filter=file_prefix_filter,
                                   file_exclude_filter=file_exclude_filter,
                                   file_install_tag=file_install_tag,

legendary/downloader/mp/manager.py

@@ -80,8 +80,10 @@ def __init__(self, download_dir, base_url, cache_dir=None, status_q=None,
         self.num_tasks_processed_since_last = 0
 
     def run_analysis(self, manifest: Manifest, old_manifest: Manifest = None,
-                     patch=True, resume=True, file_prefix_filter=None,
-                     file_exclude_filter=None, file_install_tag=None,
+                     patch=True, resume=True, read_files=True,
+                     file_prefix_filter=None,
+                     file_exclude_filter=None,
+                     file_install_tag=None,
                      processing_optimization=False) -> AnalysisResult:
         """
         Run analysis on manifest and old manifest (if not None) and return a result
@@ -90,6 +92,7 @@ def run_analysis(self, manifest: Manifest, old_manifest: Manifest = None,
         :param manifest: Manifest to install
         :param old_manifest: Old manifest to patch from (if applicable)
         :param patch: Patch instead of redownloading the entire file
+        :param read_files: Allow reading from already finished files
         :param resume: Continue based on resume file if it exists
         :param file_prefix_filter: Only download files that start with this prefix
         :param file_exclude_filter: Exclude files with this prefix from download
@@ -320,25 +323,27 @@ def run_analysis(self, manifest: Manifest, old_manifest: Manifest = None,
 
         # determine whether a chunk part is currently in written files
         reusable_written = defaultdict(dict)
-        cur_written_cps = defaultdict(list)
-        for cur_file in fmlist:
-            cur_file_cps = dict()
-            cur_file_offset = 0
-            for cp in cur_file.chunk_parts:
-                key = (cp.guid_num, cp.offset, cp.size)
-                for wr_file_name, wr_file_offset, wr_cp_offset, wr_cp_end_offset in cur_written_cps[cp.guid_num]:
-                    # check if new chunk part is wholly contained in a written chunk part
-                    cur_cp_end_offset = cp.offset + cp.size
-                    if wr_cp_offset <= cp.offset and wr_cp_end_offset >= cur_cp_end_offset:
-                        references[cp.guid_num] -= 1
-                        reuse_offset = wr_file_offset + (cp.offset - wr_cp_offset)
-                        reusable_written[cur_file.filename][key] = (wr_file_name, reuse_offset)
-                        break
-                cur_file_cps[cp.guid_num] = (cur_file.filename, cur_file_offset, cp.offset, cp.offset + cp.size)
-                cur_file_offset += cp.size
+        if read_files:
+            self.log.debug('Analyzing manifest for re-usable chunks in saved files...')
+            cur_written_cps = defaultdict(list)
+            for cur_file in fmlist:
+                cur_file_cps = dict()
+                cur_file_offset = 0
+                for cp in cur_file.chunk_parts:
+                    key = (cp.guid_num, cp.offset, cp.size)
+                    for wr_file_name, wr_file_offset, wr_cp_offset, wr_cp_end_offset in cur_written_cps[cp.guid_num]:
+                        # check if new chunk part is wholly contained in a written chunk part
+                        cur_cp_end_offset = cp.offset + cp.size
+                        if wr_cp_offset <= cp.offset and wr_cp_end_offset >= cur_cp_end_offset:
+                            references[cp.guid_num] -= 1
+                            reuse_offset = wr_file_offset + (cp.offset - wr_cp_offset)
+                            reusable_written[cur_file.filename][key] = (wr_file_name, reuse_offset)
+                            break
+                    cur_file_cps[cp.guid_num] = (cur_file.filename, cur_file_offset, cp.offset, cp.offset + cp.size)
+                    cur_file_offset += cp.size
 
-            for guid, value in cur_file_cps.items():
-                cur_written_cps[guid].append(value)
+                for guid, value in cur_file_cps.items():
+                    cur_written_cps[guid].append(value)
 
         last_cache_size = current_cache_size = 0
         # set to determine whether a file is currently cached or not