Add auto downloading of project file dependencies, clean up download APIs

Author: negasora

binaryninjaapi.h

@@ -22249,13 +22249,21 @@ namespace BinaryNinja::Collaboration
 		 */
 		void DeleteSnapshot(const Ref<CollabSnapshot> snapshot);
 
+		/*!
+		    Download a remote file and possibly dependencies to its project
+			Dependency download behavior depends on the value of the collaboration.autoDownloadFileDependencies setting
+		    \param progress Function to call on progress updates
+		    \throws RemoteException If there is an error in any request or if the remote is not connected
+		 */
+		void Download(ProgressFunction progress = DefaultProgressFunction);
+
 		/*!
 		    Download the contents of a remote file
 		    \param progress Function to call on progress updates
 		    \return Contents of the file
 		    \throws RemoteException If there is an error in any request or if the remote is not connected
 		 */
-		std::vector<uint8_t> Download(ProgressFunction progress = {});
+		std::vector<uint8_t> DownloadContents(ProgressFunction progress = {});
 
 		/*!
 		    Get the current user positions for this file

binaryninjacore.h

@@ -37,14 +37,14 @@
 // Current ABI version for linking to the core. This is incremented any time
 // there are changes to the API that affect linking, including new functions,
 // new types, or modifications to existing functions or types.
-#define BN_CURRENT_CORE_ABI_VERSION 146
+#define BN_CURRENT_CORE_ABI_VERSION 147
 
 // Minimum ABI version that is supported for loading of plugins. Plugins that
 // are linked to an ABI version less than this will not be able to load and
 // will require rebuilding. The minimum version is increased when there are
 // incompatible changes that break binary compatibility, such as changes to
 // existing types or functions.
-#define BN_MINIMUM_CORE_ABI_VERSION 146
+#define BN_MINIMUM_CORE_ABI_VERSION 147
 
 #ifdef __GNUC__
 	#ifdef BINARYNINJACORE_LIBRARY
@@ -8449,7 +8449,8 @@ extern "C"
 	BINARYNINJACOREAPI bool BNRemoteFilePullSnapshots(BNRemoteFile* file, BNProgressFunction progress, void* progressContext);
 	BINARYNINJACOREAPI BNCollaborationSnapshot* BNRemoteFileCreateSnapshot(BNRemoteFile* file, const char* name, uint8_t* contents, size_t contentsSize, uint8_t* analysisCacheContents, size_t analysisCacheContentsSize, uint8_t* fileContents, size_t fileContentsSize, const char** parentIds, size_t parentIdCount, BNProgressFunction progress, void* progressContext);
 	BINARYNINJACOREAPI bool BNRemoteFileDeleteSnapshot(BNRemoteFile* file, BNCollaborationSnapshot* snapshot);
-	BINARYNINJACOREAPI bool BNRemoteFileDownload(BNRemoteFile* file, BNProgressFunction progress, void* progressCtxt, uint8_t** data, size_t* size);
+	BINARYNINJACOREAPI bool BNRemoteFileDownload(BNRemoteFile* file, BNProgressFunction progress, void* progressCtxt);
+	BINARYNINJACOREAPI bool BNRemoteFileDownloadContents(BNRemoteFile* file, BNProgressFunction progress, void* progressCtxt, uint8_t** data, size_t* size);
 	BINARYNINJACOREAPI char* BNRemoteFileRequestUserPositions(BNRemoteFile* file);
 	BINARYNINJACOREAPI char* BNRemoteFileRequestChatLog(BNRemoteFile* file);
 

collaboration.cpp

@@ -1853,14 +1853,23 @@ void RemoteFile::DeleteSnapshot(const Ref<CollabSnapshot> snapshot)
 }
 
 
-std::vector<uint8_t> RemoteFile::Download(ProgressFunction progress)
+void RemoteFile::Download(ProgressFunction progress)
+{
+	ProgressContext pctxt;
+	pctxt.callback = progress;
+	if (!BNRemoteFileDownload(m_object, ProgressCallback, &pctxt))
+		throw RemoteException("Failed to download file");
+}
+
+
+std::vector<uint8_t> RemoteFile::DownloadContents(ProgressFunction progress)
 {
 	ProgressContext pctxt;
 	pctxt.callback = progress;
 	size_t size = 0;
 	uint8_t* data;
-	if (!BNRemoteFileDownload(m_object, ProgressCallback, &pctxt, &data, &size))
-		throw SyncException("Failed to download file");
+	if (!BNRemoteFileDownloadContents(m_object, ProgressCallback, &pctxt, &data, &size))
+		throw SyncException("Failed to download file contents");
 
 	std::vector<uint8_t> out;
 	out.insert(out.end(), &data[0], &data[size]);

python/collaboration/examples/download_everything.py

@@ -19,7 +19,6 @@
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 # IN THE SOFTWARE.
 
-from pathlib import Path
 import sys
 
 import binaryninja
@@ -40,18 +39,13 @@ def main():
 		# Pull every file from every project
 		for project in remote.projects:
 			for file in project.files:
-				bndb_path = file.default_path
-				print(f"{project.name}/{file.name} BNDB at {bndb_path}")
+				print(f"{project.name}/{file.name} BNDB at {file.default_path}")
 
 				try:
-					metadata: binaryninja.FileMetadata = file.download_to_bndb(bndb_path)
-
-					for v in metadata.existing_views:
-						if v == 'Raw':
-							continue
-						bv = metadata.get_view_of_type(v)
+					file.download()
+					with binaryninja.load(file.core_file) as bv:
 						# Show the entry point to demonstrate we have pulled the analyzed file
-						print(f"{project.name}/{file.name} {v} Entrypoint @ 0x{bv.entry_point:08x}")
+						print(f"{project.name}/{file.name} {bv.view_type} Entrypoint @ 0x{bv.entry_point:08x}")
 				except InterruptedError as e:
 					# In case of ^C
 					raise e

python/collaboration/examples/multitool.py

@@ -237,7 +237,7 @@ def main():
 				print(f"Unknown File: {args.name}")
 				sys.exit(1)
 			with TqdmProgress(desc="", leave=False) as t:
-				file.download(lambda cur, max: t.progress(cur, max))
+				file.download_contents(lambda cur, max: t.progress(cur, max))
 			print_file_info(file)
 		elif args.file_command == "delete":
 			file = project.get_file_by_name(args.name)

python/collaboration/examples/sync_test.py

@@ -56,27 +56,27 @@ def main():
 			project_dir = Path(file.default_path).parent
 
 			print(f'Snapshots: {[snapshot.id for snapshot in file.snapshots]}')
-			bv = binaryninja.load(file.default_path)
-			view_type = bv.view_type
-			assert collaboration.RemoteFile.get_for_bv(bv) == file
+			with binaryninja.load(file.default_path) as bv:
+				assert collaboration.RemoteFile.get_for_bv(bv) == file
+				assert bv.entry_function is not None, "Failed to get binary entry function"
 
-			print(f'Setting entry function at 0x{bv.entry_function.start:08x} name to \'entry_function\'')
-			bv.entry_function.name = 'entry_function'
-			bv.file.save_auto_snapshot()
+				print(f'Setting entry function at 0x{bv.entry_function.start:08x} name to \'entry_function\'')
+				bv.entry_function.name = 'entry_function'
+				bv.file.save_auto_snapshot()
 
-			file.sync(bv, lambda conflicts: False)
-			print(f'Snapshots: {[snapshot.id for snapshot in file.snapshots]}')
+				file.sync(bv, lambda conflicts: False)
+				print(f'Snapshots: {[snapshot.id for snapshot in file.snapshots]}')
 
-			# Try deleting the bndb, redownload and see if the function name is preserved
-			bv.file.close()
+			# Delete the bndb, redownload and see if the function name is preserved
 			Path(file.default_path).unlink()
 
 			print(f'Redownloading {project.name}/{file.name}...')
-			metadata = file.download_to_bndb()
-			bv = metadata.get_view_of_type(view_type)
-			print(f'Entry function name: {bv.entry_function.name}')
-			assert bv.entry_function.name == 'entry_function'
-			bv.file.close()
+			file.download()
+			with binaryninja.load(file.core_file) as bv:
+				assert bv.entry_function is not None, "Failed to get binary entry function after redownload"
+
+				print(f'Entry function name: {bv.entry_function.name}')
+				assert bv.entry_function.name == 'entry_function'
 
 		finally:
 			# Clean up

python/collaboration/file.py

@@ -374,7 +374,19 @@ def delete_snapshot(self, snapshot: 'snapshot.CollabSnapshot'):
 		if not core.BNRemoteFileDeleteSnapshot(self._handle, snapshot._handle):
 			raise RuntimeError(util._last_error())
 
-	def download(self, progress: 'util.ProgressFuncType' = util.nop) -> bytes:
+	def download(self, progress: 'util.ProgressFuncType' = util.nop):
+		"""
+		Download a remote file and possibly dependencies to its project
+		Dependency download behavior depends on the value of the collaboration.autoDownloadFileDependencies setting
+
+		:param progress: Function to call on progress updates
+		:raises: RuntimeError if there was an error
+		"""
+		value = core.BNRemoteFileDownload(self._handle, util.wrap_progress(progress), None)
+		if not value:
+			raise RuntimeError(util._last_error())
+
+	def download_contents(self, progress: 'util.ProgressFuncType' = util.nop) -> bytes:
 		"""
 		Download the contents of a remote file
 
@@ -384,7 +396,7 @@ def download(self, progress: 'util.ProgressFuncType' = util.nop) -> bytes:
 		"""
 		data = (ctypes.POINTER(ctypes.c_ubyte))()
 		size = ctypes.c_size_t()
-		value = core.BNRemoteFileDownload(self._handle, util.wrap_progress(progress), None, data, size)
+		value = core.BNRemoteFileDownloadContents(self._handle, util.wrap_progress(progress), None, data, size)
 		if not value:
 			raise RuntimeError(util._last_error())
 		return bytes(ctypes.cast(data, ctypes.POINTER(ctypes.c_uint8 * size.value)).contents)
@@ -399,6 +411,7 @@ def download_to_bndb(self, path: Optional[str] = None, progress: 'util.ProgressF
 		:return: Constructed FileMetadata object
 		:raises: RuntimeError if there was an error
 		"""
+		# TODO: deprecated, use RemoteFile.download() and ProjectFile.export()
 		if path is None:
 			path = self.default_path
 		file = databasesync.download_file(self, path, util.split_progress(progress, 0, [0.5, 0.5]))

rust/src/collaboration/file.rs

@@ -394,50 +394,60 @@ impl RemoteFile {
         unsafe { BnString::into_string(result) }
     }
 
-    // TODO: AsRef<Path>
-    /// Download a file from its remote, saving all snapshots to a database in the
-    /// specified location. Returns a FileContext for opening the file later.
+    /// Download a remote file and possibly dependencies to its project
+    /// Dependency download behavior depends on the value of the collaboration.autoDownloadFileDependencies setting
     ///
-    /// * `db_path` - File path for saved database
-    /// * `progress_function` - Function to call for progress updates
-    pub fn download(&self, db_path: &Path) -> Result<Ref<FileMetadata>, ()> {
-        sync::download_file(self, db_path)
+    /// * `progress` - Function to call on progress updates
+    pub fn download(&self) -> Result<(), ()> {
+        self.download_with_progress(NoProgressCallback)
     }
 
-    // TODO: AsRef<Path>
-    /// Download a file from its remote, saving all snapshots to a database in the
-    /// specified location. Returns a FileContext for opening the file later.
+    /// Download a remote file and possibly dependencies to its project
+    /// Dependency download behavior depends on the value of the collaboration.autoDownloadFileDependencies setting
     ///
-    /// * `db_path` - File path for saved database
-    /// * `progress_function` - Function to call for progress updates
-    pub fn download_with_progress<F>(
-        &self,
-        db_path: &Path,
-        progress_function: F,
-    ) -> Result<Ref<FileMetadata>, ()>
+    /// * `progress` - Function to call on progress updates
+    pub fn download_with_progress<P>(&self, mut progress: P) -> Result<(), ()>
     where
-        F: ProgressCallback,
+        P: ProgressCallback,
     {
-        sync::download_file_with_progress(self, db_path, progress_function)
+        let success = unsafe {
+            BNRemoteFileDownload(
+                self.handle.as_ptr(),
+                Some(P::cb_progress_callback),
+                &mut progress as *mut P as *mut c_void,
+            )
+        };
+        success.then_some(()).ok_or(())
     }
 
     /// Download a remote file and save it to a BNDB at the given `path`, returning the associated [`FileMetadata`].
-    pub fn download_database(&self, path: &Path) -> Result<Ref<FileMetadata>, ()> {
-        let file = self.download(path)?;
-        let database = file.database().ok_or(())?;
-        self.sync(&database, DatabaseConflictHandlerFail, NoNameChangeset)?;
-        Ok(file)
+    /// Download a file from its remote, saving all snapshots to a database in the
+    /// specified location. Returns a FileContext for opening the file later.
+    ///
+    /// * `path` - File path for saved database
+    pub fn download_database(&self, path: impl AsRef<Path>) -> Result<Ref<FileMetadata>, ()> {
+        //TODO: deprecated, use RemoteFile.download() and ProjectFile.export()
+        self.download_database_with_progress(path, NoProgressCallback)
     }
 
-    // TODO: This might be a bad helper... maybe remove...
-    /// Download a remote file and save it to a BNDB at the given `path`.
+    /// Download a remote file and save it to a BNDB at the given `path`, returning the associated [`FileMetadata`].
+    /// Download a file from its remote, saving all snapshots to a database in the
+    /// specified location. Returns a FileContext for opening the file later.
+    ///
+    /// * `path` - File path for saved database
+    /// * `progress_function` - Function to call for progress updates
     pub fn download_database_with_progress(
         &self,
-        path: &Path,
+        path: impl AsRef<Path>,
         progress: impl ProgressCallback,
     ) -> Result<Ref<FileMetadata>, ()> {
+        //TODO: deprecated, use RemoteFile.download_with_progress() and ProjectFile.export()
         let mut progress = progress.split(&[50, 50]);
-        let file = self.download_with_progress(path, progress.next_subpart().unwrap())?;
+        let file = sync::download_file_with_progress(
+            self,
+            path.as_ref(),
+            progress.next_subpart().unwrap(),
+        )?;
         let database = file.database().ok_or(())?;
         self.sync_with_progress(
             &database,