Initial Commit
This commit is contained in:
Gregory Campbell
@@ -19,3 +19,8 @@ target
|
|||||||
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||||
#.idea/
|
#.idea/
|
||||||
|
|
||||||
|
|
||||||
|
# Added by cargo
|
||||||
|
|
||||||
|
/target
|
||||||
|
|||||||
Generated
+328
@@ -0,0 +1,328 @@
|
|||||||
|
# This file is automatically @generated by Cargo.
|
||||||
|
# It is not intended for manual editing.
|
||||||
|
version = 4
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "anstream"
|
||||||
|
version = "1.0.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
|
||||||
|
dependencies = [
|
||||||
|
"anstyle",
|
||||||
|
"anstyle-parse",
|
||||||
|
"anstyle-query",
|
||||||
|
"anstyle-wincon",
|
||||||
|
"colorchoice",
|
||||||
|
"is_terminal_polyfill",
|
||||||
|
"utf8parse",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "anstyle"
|
||||||
|
version = "1.0.14"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "anstyle-parse"
|
||||||
|
version = "1.0.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
|
||||||
|
dependencies = [
|
||||||
|
"utf8parse",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "anstyle-query"
|
||||||
|
version = "1.1.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
||||||
|
dependencies = [
|
||||||
|
"windows-sys",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "anstyle-wincon"
|
||||||
|
version = "3.0.11"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
||||||
|
dependencies = [
|
||||||
|
"anstyle",
|
||||||
|
"once_cell_polyfill",
|
||||||
|
"windows-sys",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "autocfg"
|
||||||
|
version = "1.5.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "bit-vec"
|
||||||
|
version = "0.9.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51"
|
||||||
|
dependencies = [
|
||||||
|
"serde",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "clap"
|
||||||
|
version = "4.6.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51"
|
||||||
|
dependencies = [
|
||||||
|
"clap_builder",
|
||||||
|
"clap_derive",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "clap_builder"
|
||||||
|
version = "4.6.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
|
||||||
|
dependencies = [
|
||||||
|
"anstream",
|
||||||
|
"anstyle",
|
||||||
|
"clap_lex",
|
||||||
|
"strsim",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "clap_derive"
|
||||||
|
version = "4.6.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9"
|
||||||
|
dependencies = [
|
||||||
|
"heck",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "clap_lex"
|
||||||
|
version = "1.1.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "colorchoice"
|
||||||
|
version = "1.0.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "commands"
|
||||||
|
version = "0.0.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "688226b9769bbf11a9d82a94fb4adda15793a6023d33a8ca7695dbafec6f4123"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "crossbeam-deque"
|
||||||
|
version = "0.8.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
|
||||||
|
dependencies = [
|
||||||
|
"crossbeam-epoch",
|
||||||
|
"crossbeam-utils",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "crossbeam-epoch"
|
||||||
|
version = "0.9.18"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
|
||||||
|
dependencies = [
|
||||||
|
"crossbeam-utils",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "crossbeam-utils"
|
||||||
|
version = "0.8.21"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "either"
|
||||||
|
version = "1.16.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "heck"
|
||||||
|
version = "0.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "huffman-compression-rs"
|
||||||
|
version = "0.1.0"
|
||||||
|
dependencies = [
|
||||||
|
"bit-vec",
|
||||||
|
"clap",
|
||||||
|
"commands",
|
||||||
|
"lock",
|
||||||
|
"rayon",
|
||||||
|
"rmp-serde",
|
||||||
|
"serde",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "is_terminal_polyfill"
|
||||||
|
version = "1.70.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "lock"
|
||||||
|
version = "0.1.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "058cf9c2b0daa80dafc6d65b594ea04db8050522cfbf39f5d5aef2184f2a5a55"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "num-traits"
|
||||||
|
version = "0.2.19"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
|
||||||
|
dependencies = [
|
||||||
|
"autocfg",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "once_cell_polyfill"
|
||||||
|
version = "1.70.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "proc-macro2"
|
||||||
|
version = "1.0.106"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
|
||||||
|
dependencies = [
|
||||||
|
"unicode-ident",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "quote"
|
||||||
|
version = "1.0.45"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rayon"
|
||||||
|
version = "1.12.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
|
||||||
|
dependencies = [
|
||||||
|
"either",
|
||||||
|
"rayon-core",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rayon-core"
|
||||||
|
version = "1.13.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
|
||||||
|
dependencies = [
|
||||||
|
"crossbeam-deque",
|
||||||
|
"crossbeam-utils",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rmp"
|
||||||
|
version = "0.8.15"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c"
|
||||||
|
dependencies = [
|
||||||
|
"num-traits",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rmp-serde"
|
||||||
|
version = "1.3.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155"
|
||||||
|
dependencies = [
|
||||||
|
"rmp",
|
||||||
|
"serde",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde"
|
||||||
|
version = "1.0.228"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
|
||||||
|
dependencies = [
|
||||||
|
"serde_core",
|
||||||
|
"serde_derive",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde_core"
|
||||||
|
version = "1.0.228"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
|
||||||
|
dependencies = [
|
||||||
|
"serde_derive",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde_derive"
|
||||||
|
version = "1.0.228"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "strsim"
|
||||||
|
version = "0.11.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "syn"
|
||||||
|
version = "2.0.118"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"unicode-ident",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "unicode-ident"
|
||||||
|
version = "1.0.24"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "utf8parse"
|
||||||
|
version = "0.2.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-link"
|
||||||
|
version = "0.2.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-sys"
|
||||||
|
version = "0.61.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
|
||||||
|
dependencies = [
|
||||||
|
"windows-link",
|
||||||
|
]
|
||||||
+13
@@ -0,0 +1,13 @@
|
|||||||
|
[package]
|
||||||
|
name = "huffman-compression-rs"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition = "2024"
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
bit-vec = { version = "0.9.1", features = ["serde"] }
|
||||||
|
clap = { version = "4.6.1", features = ["derive"] }
|
||||||
|
commands = "0.0.5"
|
||||||
|
lock = "0.1.0"
|
||||||
|
rayon = "1.12.0"
|
||||||
|
rmp-serde = "1.3.1"
|
||||||
|
serde = { version = "1.0.228", features = ["derive"] }
|
||||||
@@ -0,0 +1,166 @@
|
|||||||
|
use bit_vec::BitVec;
|
||||||
|
use rayon::prelude::*;
|
||||||
|
use rmp_serde;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use std::{collections::HashMap, hash::Hash};
|
||||||
|
|
||||||
|
use crate::huffman::{self, Tree};
|
||||||
|
use Tree::*;
|
||||||
|
|
||||||
|
#[derive(serde::Serialize, serde::Deserialize)]
|
||||||
|
struct CompressedData<T: Eq + Hash> {
|
||||||
|
encoder: HashMap<T, BitVec>,
|
||||||
|
data: Vec<BitVec>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn compress<'a, T, FreqsF, TokenExtractor, TokensIter>(
|
||||||
|
lines: &'a [String],
|
||||||
|
get_freqs: FreqsF,
|
||||||
|
line_to_tokens: TokenExtractor,
|
||||||
|
) -> Result<Vec<u8>, Box<dyn std::error::Error>>
|
||||||
|
where
|
||||||
|
T: Clone + Eq + Hash + Send + Sync + Serialize,
|
||||||
|
FreqsF: Fn(&'a [String]) -> HashMap<T, u64>,
|
||||||
|
TokenExtractor: Fn(&'a str) -> TokensIter + Send + Sync,
|
||||||
|
TokensIter: Iterator<Item = T>,
|
||||||
|
{
|
||||||
|
let freqs = get_freqs(lines);
|
||||||
|
let tree = huffman::huffman_tree(&freqs);
|
||||||
|
let encoder = tree.to_encoder();
|
||||||
|
|
||||||
|
let data = lines
|
||||||
|
.par_iter()
|
||||||
|
.map(|line| {
|
||||||
|
let mut bits = BitVec::new();
|
||||||
|
for token in line_to_tokens(line) {
|
||||||
|
bits.extend(encoder.get(&token).unwrap().iter());
|
||||||
|
}
|
||||||
|
bits
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
let compressed_data = CompressedData { encoder, data };
|
||||||
|
rmp_serde::encode::to_vec(&compressed_data).map_err(|err| err.into())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn extract<'a, T, F>(
|
||||||
|
data: &'a [u8],
|
||||||
|
tokens_to_line: F,
|
||||||
|
) -> Result<Vec<String>, Box<dyn std::error::Error>>
|
||||||
|
where
|
||||||
|
T: Clone + Eq + Hash + Send + Sync + Deserialize<'a>,
|
||||||
|
F: Fn(Vec<T>) -> String + Send + Sync,
|
||||||
|
{
|
||||||
|
let CompressedData { encoder, data }: CompressedData<T> = rmp_serde::decode::from_slice(data)?;
|
||||||
|
|
||||||
|
let decoder = build_decoder(&encoder);
|
||||||
|
let lines = data
|
||||||
|
.par_iter()
|
||||||
|
.map(|line| {
|
||||||
|
let mut tokens = Vec::new();
|
||||||
|
let mut node = &decoder;
|
||||||
|
|
||||||
|
for bit in line.iter() {
|
||||||
|
node = if bit {
|
||||||
|
node.right.as_ref().expect("invalid Huffman bit sequence")
|
||||||
|
} else {
|
||||||
|
node.left.as_ref().expect("invalid Huffman bit sequence")
|
||||||
|
};
|
||||||
|
|
||||||
|
if let Some(token) = &node.token {
|
||||||
|
tokens.push(token.clone());
|
||||||
|
node = &decoder;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tokens_to_line(tokens)
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
Ok(lines)
|
||||||
|
}
|
||||||
|
|
||||||
|
struct DecoderNode<T> {
|
||||||
|
token: Option<T>,
|
||||||
|
left: Option<Box<DecoderNode<T>>>,
|
||||||
|
right: Option<Box<DecoderNode<T>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T> Default for DecoderNode<T> {
|
||||||
|
fn default() -> Self {
|
||||||
|
DecoderNode {
|
||||||
|
token: None,
|
||||||
|
left: None,
|
||||||
|
right: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_decoder<T: Clone + Eq + Hash>(encoder: &HashMap<T, BitVec>) -> DecoderNode<T> {
|
||||||
|
let mut root = DecoderNode::default();
|
||||||
|
|
||||||
|
for (token, code) in encoder {
|
||||||
|
let mut node = &mut root;
|
||||||
|
for bit in code.iter() {
|
||||||
|
node = if bit {
|
||||||
|
node.right.get_or_insert_with(|| Box::new(DecoderNode::default()))
|
||||||
|
} else {
|
||||||
|
node.left.get_or_insert_with(|| Box::new(DecoderNode::default()))
|
||||||
|
};
|
||||||
|
}
|
||||||
|
node.token = Some(token.clone());
|
||||||
|
}
|
||||||
|
|
||||||
|
root
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T: Eq + Clone + Hash> Tree<T> {
|
||||||
|
pub fn to_encoder(&self) -> HashMap<T, BitVec> {
|
||||||
|
let mut encoder = HashMap::new();
|
||||||
|
|
||||||
|
let mut stack = vec![(self, BitVec::new())];
|
||||||
|
while let Some((node, path)) = stack.pop() {
|
||||||
|
match node {
|
||||||
|
Leaf { token, .. } => {
|
||||||
|
encoder.insert(token.clone(), path.clone());
|
||||||
|
}
|
||||||
|
Node { left, right, .. } => {
|
||||||
|
let mut left_path = path.clone();
|
||||||
|
left_path.push(false);
|
||||||
|
stack.push((left, left_path));
|
||||||
|
|
||||||
|
let mut right_path = path.clone();
|
||||||
|
right_path.push(true);
|
||||||
|
stack.push((right, right_path));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
encoder
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::freq::{char_frequencies, word_frequencies};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn compress_decompress_test() {
|
||||||
|
let lines = vec![
|
||||||
|
"hey there! nice to meet you.".to_string(),
|
||||||
|
"Serde is a framework for serializing and deserializing Rust data structures"
|
||||||
|
.to_string(),
|
||||||
|
];
|
||||||
|
|
||||||
|
let data = compress(&lines, char_frequencies, |line| line.chars()).unwrap();
|
||||||
|
let res_lines = extract(&data, |x: Vec<char>| x.into_iter().collect()).unwrap();
|
||||||
|
assert_eq!(&lines, &res_lines);
|
||||||
|
|
||||||
|
let data = compress(&lines, word_frequencies, |line| {
|
||||||
|
line.split_ascii_whitespace().map(|token| token.to_string())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
let res_lines = extract(&data, |x: Vec<String>| x.join(" ")).unwrap();
|
||||||
|
assert_eq!(&lines, &res_lines);
|
||||||
|
}
|
||||||
|
}
|
||||||
+48
@@ -0,0 +1,48 @@
|
|||||||
|
use rayon::prelude::*;
|
||||||
|
use std::collections::HashMap;
|
||||||
|
|
||||||
|
pub fn char_frequencies(lines: &[String]) -> HashMap<char, u64> {
|
||||||
|
lines
|
||||||
|
.par_iter()
|
||||||
|
.fold(
|
||||||
|
HashMap::new,
|
||||||
|
|mut freqs: HashMap<_, _>, line: &String| {
|
||||||
|
for ch in line.chars() {
|
||||||
|
*freqs.entry(ch).or_insert(0) += 1;
|
||||||
|
}
|
||||||
|
freqs
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.reduce(
|
||||||
|
HashMap::new,
|
||||||
|
|mut freqs1, freqs2| {
|
||||||
|
freqs2
|
||||||
|
.into_iter()
|
||||||
|
.for_each(|(ch, n)| *freqs1.entry(ch).or_insert(0) += n);
|
||||||
|
freqs1
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn word_frequencies(lines: &[String]) -> HashMap<String, u64> {
|
||||||
|
lines
|
||||||
|
.par_iter()
|
||||||
|
.fold(
|
||||||
|
HashMap::new,
|
||||||
|
|mut freqs: HashMap<_, _>, line: &String| {
|
||||||
|
for word in line.split_ascii_whitespace() {
|
||||||
|
*freqs.entry(word.to_string()).or_insert(0) += 1;
|
||||||
|
}
|
||||||
|
freqs
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.reduce(
|
||||||
|
HashMap::new,
|
||||||
|
|mut freqs1, freqs2| {
|
||||||
|
freqs2
|
||||||
|
.into_iter()
|
||||||
|
.for_each(|(word, n)| *freqs1.entry(word).or_insert(0) += n);
|
||||||
|
freqs1
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
+158
@@ -0,0 +1,158 @@
|
|||||||
|
use std::{
|
||||||
|
cmp::Reverse,
|
||||||
|
collections::{BinaryHeap, HashMap},
|
||||||
|
};
|
||||||
|
use Tree::*;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub enum Tree<T> {
|
||||||
|
Leaf {
|
||||||
|
freq: u64,
|
||||||
|
token: T,
|
||||||
|
},
|
||||||
|
Node {
|
||||||
|
freq: u64,
|
||||||
|
left: Box<Tree<T>>,
|
||||||
|
right: Box<Tree<T>>,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
|
impl<T: Clone> Tree<T> {
|
||||||
|
pub fn freq(&self) -> u64 {
|
||||||
|
match self {
|
||||||
|
Leaf { freq, .. } => *freq,
|
||||||
|
Node { freq, .. } => *freq,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn token(&self) -> Option<T> {
|
||||||
|
match self {
|
||||||
|
Leaf { token, .. } => Some(token.clone()),
|
||||||
|
Node { .. } => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn left(&self) -> Option<&Tree<T>> {
|
||||||
|
match self {
|
||||||
|
Node { left, .. } => Some(left),
|
||||||
|
Leaf { .. } => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn right(&self) -> Option<&Tree<T>> {
|
||||||
|
match self {
|
||||||
|
Node { right, .. } => Some(right),
|
||||||
|
Leaf { .. } => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T: Clone + Eq> Ord for Tree<T> {
|
||||||
|
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
|
||||||
|
self.freq().cmp(&other.freq())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T: Clone + Eq> PartialOrd for Tree<T> {
|
||||||
|
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
|
||||||
|
Some(self.cmp(other))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn huffman_tree<T: Eq + Clone>(freqs: &HashMap<T, u64>) -> Tree<T> {
|
||||||
|
let mut heap = BinaryHeap::new();
|
||||||
|
for (token, freq) in freqs {
|
||||||
|
let (freq, token) = (*freq, token.clone());
|
||||||
|
heap.push(Reverse(Leaf { freq, token }))
|
||||||
|
}
|
||||||
|
|
||||||
|
while heap.len() > 1 {
|
||||||
|
let node1 = heap.pop().unwrap().0;
|
||||||
|
let node2 = heap.pop().unwrap().0;
|
||||||
|
|
||||||
|
let merged_node = Node {
|
||||||
|
freq: node1.freq() + node2.freq(),
|
||||||
|
left: Box::new(node1),
|
||||||
|
right: Box::new(node2),
|
||||||
|
};
|
||||||
|
heap.push(Reverse(merged_node));
|
||||||
|
}
|
||||||
|
|
||||||
|
heap.pop().unwrap().0
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::freq::char_frequencies;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn learn_frequencies_test() {
|
||||||
|
let input = vec!["this is an epic sentence".to_string(), "xyz ".to_string()];
|
||||||
|
let freqs = char_frequencies(&input);
|
||||||
|
assert_eq!(freqs[&' '], 5);
|
||||||
|
assert_eq!(freqs[&'t'], 2);
|
||||||
|
assert_eq!(freqs[&'i'], 3);
|
||||||
|
assert_eq!(freqs[&'p'], 1);
|
||||||
|
assert_eq!(freqs[&'z'], 1);
|
||||||
|
assert_eq!(freqs.keys().len(), 13);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn huffman_tree_test() {
|
||||||
|
let mut freqs = HashMap::new();
|
||||||
|
freqs.insert('a', 40);
|
||||||
|
freqs.insert('b', 35);
|
||||||
|
freqs.insert('c', 20);
|
||||||
|
freqs.insert('d', 5);
|
||||||
|
|
||||||
|
let tree = huffman_tree(&freqs);
|
||||||
|
assert_eq!(tree.freq(), 100);
|
||||||
|
|
||||||
|
// the most frequent character only requires 1 bit
|
||||||
|
assert_eq!(tree.left().and_then(|n| n.token()), Some('a'));
|
||||||
|
assert_eq!(tree.left().map(|n| n.freq()), Some(40));
|
||||||
|
|
||||||
|
// the second most frequent character requires 2 bits
|
||||||
|
assert_eq!(
|
||||||
|
tree.right().and_then(|t| t.right()).and_then(|n| n.token()),
|
||||||
|
Some('b')
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
tree.right().and_then(|t| t.right()).map(|n| n.freq()),
|
||||||
|
Some(35)
|
||||||
|
);
|
||||||
|
|
||||||
|
// the least frequent characters require 3 bits
|
||||||
|
assert_eq!(
|
||||||
|
tree.right()
|
||||||
|
.and_then(|t| t.left())
|
||||||
|
.and_then(|t| t.left())
|
||||||
|
.and_then(|n| n.token()),
|
||||||
|
Some('d')
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
tree.right()
|
||||||
|
.and_then(|t| t.left())
|
||||||
|
.and_then(|t| t.left())
|
||||||
|
.map(|n| n.freq()),
|
||||||
|
Some(5)
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
tree.right()
|
||||||
|
.and_then(|t| t.left())
|
||||||
|
.and_then(|t| t.right())
|
||||||
|
.and_then(|n| n.token()),
|
||||||
|
Some('c')
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
tree.right()
|
||||||
|
.and_then(|t| t.left())
|
||||||
|
.and_then(|t| t.right())
|
||||||
|
.map(|n| n.freq()),
|
||||||
|
Some(20)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
+106
@@ -0,0 +1,106 @@
|
|||||||
|
mod compression;
|
||||||
|
mod freq;
|
||||||
|
mod huffman;
|
||||||
|
|
||||||
|
use clap::Parser;
|
||||||
|
use clap::ValueEnum;
|
||||||
|
use std::fs::{self, File};
|
||||||
|
use std::io::Write;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::time;
|
||||||
|
|
||||||
|
#[derive(Parser, Debug)]
|
||||||
|
#[command(version, about, long_about = None)]
|
||||||
|
struct Args {
|
||||||
|
#[arg(value_enum)]
|
||||||
|
action: Action,
|
||||||
|
#[arg(value_enum)]
|
||||||
|
mode: Mode,
|
||||||
|
input: PathBuf,
|
||||||
|
output: PathBuf,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
|
||||||
|
enum Action {
|
||||||
|
Compress,
|
||||||
|
Extract,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
|
||||||
|
enum Mode {
|
||||||
|
Words,
|
||||||
|
Chars,
|
||||||
|
}
|
||||||
|
|
||||||
|
// cargo run --release -- compress words data/wikisent2.txt data/words.huffman
|
||||||
|
// cargo run --release -- extract words data/words.huffman data/extracted.txt
|
||||||
|
// (to check if the extraction was correct: `diff data/wikisent2.txt data/extracted.txt`)
|
||||||
|
//
|
||||||
|
// cargo run --release -- compress chars data/wikisent2.txt data/chars.huffman
|
||||||
|
// cargo run --release -- extract chars data/chars.huffman data/extracted.txt
|
||||||
|
//
|
||||||
|
// to compare with zip:
|
||||||
|
// time zip data/test.zip data/wikisent2.txt
|
||||||
|
// time unzip data/test.zip -d data/test_zip
|
||||||
|
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
let args = Args::parse();
|
||||||
|
|
||||||
|
match args.action {
|
||||||
|
Action::Compress => {
|
||||||
|
let timer = time::Instant::now();
|
||||||
|
let text = fs::read_to_string(args.input)?;
|
||||||
|
let lines: Vec<_> = text.split('\n').map(|x| x.to_string()).collect();
|
||||||
|
let time = timer.elapsed();
|
||||||
|
let lines_count = lines.len();
|
||||||
|
println!("Read the source file with {lines_count} lines in {time:?}");
|
||||||
|
|
||||||
|
let timer = time::Instant::now();
|
||||||
|
let compressed = match args.mode {
|
||||||
|
Mode::Words => compression::compress(&lines, freq::word_frequencies, |line| {
|
||||||
|
line.split_ascii_whitespace().map(|token| token.to_string())
|
||||||
|
}),
|
||||||
|
Mode::Chars => {
|
||||||
|
compression::compress(&lines, freq::char_frequencies, |line| line.chars())
|
||||||
|
}
|
||||||
|
}?;
|
||||||
|
let time = timer.elapsed();
|
||||||
|
println!("Compressed as {mode:?} in {time:?}.", mode = args.mode);
|
||||||
|
|
||||||
|
let timer = time::Instant::now();
|
||||||
|
let mut out_f = File::create(&args.output)?;
|
||||||
|
out_f.write_all(&compressed)?;
|
||||||
|
let time = timer.elapsed();
|
||||||
|
println!(
|
||||||
|
"Wrote to {output_path:?} in {time:?}",
|
||||||
|
output_path = args.output
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Action::Extract => {
|
||||||
|
let timer = time::Instant::now();
|
||||||
|
let data = fs::read(&args.input)?;
|
||||||
|
let time = timer.elapsed();
|
||||||
|
println!("Read the compressed file in {time:?}");
|
||||||
|
|
||||||
|
let timer = time::Instant::now();
|
||||||
|
let content = match args.mode {
|
||||||
|
Mode::Words => compression::extract(&data, |tokens: Vec<String>| tokens.join(" "))?,
|
||||||
|
Mode::Chars => {
|
||||||
|
compression::extract(&data, |tokens: Vec<char>| tokens.into_iter().collect())?
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let time = timer.elapsed();
|
||||||
|
let lines_count = content.len();
|
||||||
|
println!("Extracted file with {lines_count} lines in {time:?}.");
|
||||||
|
|
||||||
|
let timer = time::Instant::now();
|
||||||
|
fs::write(&args.output, content.join("\n"))?;
|
||||||
|
let time = timer.elapsed();
|
||||||
|
println!(
|
||||||
|
"Wrote to {output_path:?} in {time:?}",
|
||||||
|
output_path = args.output
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user