Merge pull request #1 from dapper91/dev

Dev

Author: dapper91

.github/workflows/release.yml

@@ -0,0 +1,18 @@
+name: release
+
+on:
+  release:
+    types:
+      - released
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install Rust
+      run: rustup update stable
+    - name: Build and publish
+      run: |
+        cargo login ${{ secrets.CRATESIO_TOKEN }}
+        cargo publish

.github/workflows/test.yml

@@ -0,0 +1,21 @@
+name: test
+
+on:
+  pull_request:
+    branches:
+      - dev
+      - master
+  push:
+    branches:
+      - master
+
+jobs:
+  test:
+    name: Run unit-tests
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install Rust
+      run: rustup update stable
+    - name: Run tests
+      run: cargo test --all-features

.gitignore

@@ -8,3 +8,5 @@ Cargo.lock
 
 # These are backup files generated by rustfmt
 **/*.rs.bk
+
+.idea

Cargo.toml

@@ -0,0 +1,35 @@
+[package]
+name = "ext-sort"
+version = "0.1.0"
+edition = "2021"
+license = "Unlicense"
+description = "rust external sort algorithm implementation"
+readme = "README.md"
+
+homepage = "https://github.com/dapper91/ext-sort-rs"
+documentation = "https://docs.rs/ext-sort/"
+repository = "https://github.com/dapper91/ext-sort-rs"
+
+categories = ["algorithms"]
+keywords = ["algorithms", "sort", "sorting", "external-sort", "external"]
+
+[dependencies]
+bytesize = { version = "^1.1", optional = true }
+deepsize = { version = "^0.2", optional = true }
+env_logger = { version = "^0.9", optional = true}
+log = "0.4"
+rayon = "^1.5"
+rmp-serde = "^0.15"
+serde = { version = "^1.0", features = ["derive"] }
+tempfile = "^3.2"
+
+[dev-dependencies]
+rstest = "^0.12"
+rand = "^0.8"
+
+[features]
+memory-limit = ["deepsize"]
+
+[[example]]
+name = "quickstart"
+required-features = ["bytesize", "env_logger"]

README.md

@@ -0,0 +1,57 @@
+# Rust external sort
+
+`ext-sort` is a rust external sort algorithm implementation.
+
+External sort algorithm implementation. External sorting is a class of sorting algorithms
+that can handle massive amounts of data. External sorting is required when the data being
+sorted do not fit into the main memory (RAM) of a computer and instead must be resided in
+slower external memory, usually a hard disk drive. Sorting is achieved in two passes.
+During the first pass it sorts chunks of data that each fit in RAM, during the second pass
+it merges the sorted chunks together.
+For more information see https://en.wikipedia.org/wiki/External_sorting.
+
+## Features
+
+* **Data agnostic:**
+  `ext-sort` support all data types that that implement `serde` serialization/deserialization.
+* **Serialization format agnostic:**
+  `ext-sort` use `MessagePack` serialization format by default, but it can be easily substituted by your custom one
+   if `MessagePack` serialization/deserialization performance is not sufficient for your task.
+* **Multithreading support:**
+  `ext-sort` support multithreading, which means data is sorted in multiple threads utilizing maximum CPU resources
+  and reducing sorting time.
+
+# Basic example
+
+``` rust
+    use std::fs;
+    use std::io::{self, prelude::*};
+    use std::path;
+    
+    use bytesize::MB;
+    use env_logger;
+    use log;
+    
+    use ext_sort::buffer::mem::MemoryLimitedBufferBuilder;
+    use ext_sort::{ExternalSorter, ExternalSorterBuilder};
+    
+    fn main() {
+        env_logger::Builder::new().filter_level(log::LevelFilter::Debug).init();
+    
+        let input_reader = io::BufReader::new(fs::File::open("input.txt").unwrap());
+        let mut output_writer = io::BufWriter::new(fs::File::create("output.txt").unwrap());
+    
+        let sorter: ExternalSorter<String, io::Error, MemoryLimitedBufferBuilder> = ExternalSorterBuilder::new()
+            .with_tmp_dir(path::Path::new("tmp"))
+            .with_buffer(MemoryLimitedBufferBuilder::new(50 * MB))
+            .build()
+            .unwrap();
+    
+        let sorted = sorter.sort(input_reader.lines()).unwrap();
+    
+        for item in sorted.map(Result::unwrap) {
+            output_writer.write_all(format!("{}\n", item).as_bytes()).unwrap();
+        }
+        output_writer.flush().unwrap();
+    }
+```

examples/quickstart.rs

@@ -0,0 +1,30 @@
+use std::fs;
+use std::io::{self, prelude::*};
+use std::path;
+
+use bytesize::MB;
+use env_logger;
+use log;
+
+use ext_sort::buffer::mem::MemoryLimitedBufferBuilder;
+use ext_sort::{ExternalSorter, ExternalSorterBuilder};
+
+fn main() {
+    env_logger::Builder::new().filter_level(log::LevelFilter::Debug).init();
+
+    let input_reader = io::BufReader::new(fs::File::open("input.txt").unwrap());
+    let mut output_writer = io::BufWriter::new(fs::File::create("output.txt").unwrap());
+
+    let sorter: ExternalSorter<String, io::Error, MemoryLimitedBufferBuilder> = ExternalSorterBuilder::new()
+        .with_tmp_dir(path::Path::new("tmp"))
+        .with_buffer(MemoryLimitedBufferBuilder::new(50 * MB))
+        .build()
+        .unwrap();
+
+    let sorted = sorter.sort(input_reader.lines()).unwrap();
+
+    for item in sorted.map(Result::unwrap) {
+        output_writer.write_all(format!("{}\n", item).as_bytes()).unwrap();
+    }
+    output_writer.flush().unwrap();
+}

rustfmt.toml

@@ -0,0 +1,2 @@
+edition = "2018"
+max_width = 120