avoid rebuilding unchanged image files

6 years ago · 7cb1bb11b2
parent 7d1d36ac8d
commit 7cb1bb11b2
5 changed files with 182 additions and 10 deletions
--- a/.gitignore
+++ b/.gitignore
@ -1,3 +1,4 @@
 /target
 **/*.rs.bk
 .idea/
+.hashes.txt
--- a/Cargo.lock
+++ b/Cargo.lock
@ -46,6 +46,14 @@ dependencies = [
 "libc 0.2.47 (registry+https://github.com/rust-lang/crates.io-index)",
 ]

+[[package]]
+name = "base64"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "byteorder 1.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "bitflags"
 version = "0.7.0"
@ -56,10 +64,23 @@ name = "bitflags"
 version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"

+[[package]]
+name = "blake2"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "byte-tools 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "crypto-mac 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "digest 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "opaque-debug 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "bread"
 version = "0.1.0"
 dependencies = [
+ "base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "blake2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
 "chrono 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
 "image 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)",
 "image-utils 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -68,6 +89,11 @@ dependencies = [
 "rss 1.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]

+[[package]]
+name = "byte-tools"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "byteorder"
 version = "1.2.7"
@ -129,6 +155,15 @@ dependencies = [
 "cfg-if 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
 ]

+[[package]]
+name = "crypto-mac"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "generic-array 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "subtle 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "darling"
 version = "0.8.3"
@ -192,6 +227,14 @@ dependencies = [
 "syn 0.15.26 (registry+https://github.com/rust-lang/crates.io-index)",
 ]

+[[package]]
+name = "digest"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "generic-array 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "either"
 version = "1.5.0"
@ -238,6 +281,14 @@ name = "fnv"
 version = "1.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"

+[[package]]
+name = "generic-array"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "gif"
 version = "0.9.2"
@ -460,6 +511,11 @@ dependencies = [
 "libc 0.2.47 (registry+https://github.com/rust-lang/crates.io-index)",
 ]

+[[package]]
+name = "opaque-debug"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "percent-encoding"
 version = "1.0.1"
@ -600,6 +656,11 @@ name = "scopeguard"
 version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"

+[[package]]
+name = "subtle"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "syn"
 version = "0.15.26"
@ -650,6 +711,11 @@ dependencies = [
 "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
 ]

+[[package]]
+name = "typenum"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "ucd-util"
 version = "0.1.3"
@ -691,8 +757,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a6d640bee2da49f60a4068a7fae53acde8982514ab7bae8b8cea9e88cbcfd799"
 "checksum backtrace 0.3.13 (registry+https://github.com/rust-lang/crates.io-index)" = "b5b493b66e03090ebc4343eb02f94ff944e0cbc9ac6571491d170ba026741eb5"
 "checksum backtrace-sys 0.1.28 (registry+https://github.com/rust-lang/crates.io-index)" = "797c830ac25ccc92a7f8a7b9862bde440715531514594a6154e3d4a54dd769b6"
+"checksum base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e"
 "checksum bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aad18937a628ec6abcd26d1489012cc0e18c21798210f491af69ded9b881106d"
 "checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12"
+"checksum blake2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "91721a6330935673395a0607df4d49a9cb90ae12d259f1b3e0a3f6e1d486872e"
+"checksum byte-tools 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "980479e6fde23246dfb54d47580d66b4e99202e7579c5eaa9fe10ecb5ebd2182"
 "checksum byteorder 1.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "94f88df23a25417badc922ab0f5716cc1330e87f71ddd9203b3a3ccd9cedf75d"
 "checksum cc 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4a8b715cb4597106ea87c7c84b2f1d452c7492033765df7f32651e66fcf749"
 "checksum cfg-if 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "082bb9b28e00d3c9d39cc03e64ce4cea0f1bb9b3fde493f0cbc008472d22bdf4"
@ -701,18 +770,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum crossbeam-deque 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f739f8c5363aca78cfb059edf753d8f0d36908c348f3d8d1503f03d8b75d9cf3"
 "checksum crossbeam-epoch 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "927121f5407de9956180ff5e936fe3cf4324279280001cd56b669d28ee7e9150"
 "checksum crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2760899e32a1d58d5abb31129f8fae5de75220bc2176e77ff7c627ae45c918d9"
+"checksum crypto-mac 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4434400df11d95d556bac068ddfedd482915eb18fe8bea89bc80b6e4b1c179e5"
 "checksum darling 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)" = "540f3246eaeecb3fc7ff9ac258f8958a57d3d87a637466fd5ad077663eabae80"
 "checksum darling_core 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)" = "69e5f4498f3c21a65d0477542d691dac79b0f44048f5174901fdbf1b676b94b7"
 "checksum darling_macro 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b8c04b3c08354a7d8a70d633931852de9e11b664881d8dd4d686e6bc7761e7fd"
 "checksum deflate 0.7.19 (registry+https://github.com/rust-lang/crates.io-index)" = "8a6abb26e16e8d419b5c78662aa9f82857c2386a073da266840e474d5055ec86"
 "checksum derive_builder 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "15d9e4f0be540b522e95c1de6200be0b12946fdd8408c093a1948de638e16f55"
 "checksum derive_builder_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e1cffc940f53a89045824e676302b840a5a60d447560704d352316e2039125a2"
+"checksum digest 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "05f47366984d3ad862010e22c7ce81a7dbcaebbdfb37241a620f8b6596ee135c"
 "checksum either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3be565ca5c557d7f59e7cfcf1844f9e3033650c929c6566f511e8005f205c1d0"
 "checksum encoding_rs 0.8.14 (registry+https://github.com/rust-lang/crates.io-index)" = "a69d152eaa438a291636c1971b0a370212165ca8a75759eb66818c5ce9b538f7"
 "checksum enum_primitive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "be4551092f4d519593039259a9ed8daedf0da12e5109c5280338073eaeb81180"
 "checksum failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "795bd83d3abeb9220f257e597aa0080a508b27533824adf336529648f6abf7e2"
 "checksum failure_derive 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "ea1063915fd7ef4309e222a5a07cf9c319fb9c7836b1f89b85458672dbb127e1"
 "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
+"checksum generic-array 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3c0f28c2f5bfb5960175af447a2da7c18900693738343dc896ffbcabd9839592"
 "checksum gif 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dd4bca55ac1f213920ce3527ccd62386f1f15fa3f1714aeee1cf93f2c416903f"
 "checksum gif 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e2e41945ba23db3bf51b24756d73d81acb4f28d85c3dccc32c6fae904438c25f"
 "checksum gif-dispose 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4c14b308f5d2295408f2bddc6628aec17382486357e7d9aed7fcd76e30a11c41"
@ -740,6 +812,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum num-traits 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)" = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31"
 "checksum num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0b3a5d7cc97d6d30d8b9bc8fa19bf45349ffe46241e8816f50f62f6d6aaabee1"
 "checksum num_cpus 1.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5a69d464bdc213aaaff628444e99578ede64e9c854025aa43b9796530afa9238"
+"checksum opaque-debug 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "51ecbcb821e1bd256d456fe858aaa7f380b63863eab2eb86eee1bd9f33dd6682"
 "checksum percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831"
 "checksum pipeline 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d15b6607fa632996eb8a17c9041cb6071cb75ac057abd45dece578723ea8c7c0"
 "checksum png 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9adebf7fb91ccf5eac9da1a8e00e83cb8ae882c3e8d8e4ad59da73cb8c82a2c9"
@ -758,11 +831,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum safe-transmute 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9604873ffe1980bc1f179103704a65c8aca141c248d9e52b7af95ff10578166e"
 "checksum scoped_threadpool 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "1d51f5df5af43ab3f1360b429fa5e0152ac5ce8c0bd6485cae490332e96846a8"
 "checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27"
+"checksum subtle 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2d67a5a62ba6e01cb2192ff309324cb4875d0c451d55fe2319433abe7a05a8ee"
 "checksum syn 0.15.26 (registry+https://github.com/rust-lang/crates.io-index)" = "f92e629aa1d9c827b2bb8297046c1ccffc57c99b947a680d3ccff1f136a3bee9"
 "checksum synstructure 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "73687139bf99285483c96ac0add482c3776528beac1d97d444f6e91f203a2015"
 "checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b"
 "checksum tiff 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a2cc6c4fd13cb1cfd20abdb196e794ceccb29371855b7e7f575945f920a5b3c2"
 "checksum time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f"
+"checksum typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169"
 "checksum ucd-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "535c204ee4d8434478593480b8f86ab45ec9aae0e83c568ca81abf0fd0e88f86"
 "checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc"
 "checksum utf8-ranges 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "796f7e48bef87609f7ade7e06495a87d5cd06c7866e6a5cbfceffc558a243737"
--- a/Cargo.toml
+++ b/Cargo.toml
@ -11,3 +11,5 @@ rss = "1.6.1"
 percent-encoding = "1.0.1"
 image-utils = "0.2.0"
 image = "0.21.0"
+blake2 = "0.8.0"
+base64 = "0.10.1"
--- a/src/hash_dict.rs
+++ b/src/hash_dict.rs
@ -0,0 +1,64 @@
+use std::collections::HashMap;
+use std::fs::File;
+use std::io::Read;
+use std::io::Write;
+use std::io;
+use std::path::PathBuf;
+
+// file-stored hash map used to prevent needless image regenerating
+#[derive(Debug)]
+pub struct HashDict {
+    hashes: HashMap<String, String>,
+    path: PathBuf,
+    any_change: bool,
+}
+
+impl HashDict {
+    pub fn load(path: PathBuf) -> Result<HashDict, io::Error>  {
+        let mut hd = HashDict {
+            hashes: HashMap::new(),
+            path,
+            any_change: false,
+        };
+
+        if !hd.path.exists() {
+            return Ok(hd);
+        }
+
+        let mut f = File::open(&hd.path)?;
+        let mut s = String::new();
+        f.read_to_string(&mut s)?;
+
+        let lines: Vec<&str> = s.split("\n").collect();
+        for l in lines {
+            let halves: Vec<&str> = l.split("\t").collect();
+            if halves.len() == 2 {
+                hd.hashes.insert(halves[0].to_string(), halves[1].to_string());
+            }
+        }
+
+        Ok(hd)
+    }
+
+    pub fn get(&self, key : &str) -> Option<&String> {
+        self.hashes.get(key)
+    }
+
+    pub fn put(&mut self, key : String, value: String) {
+        self.hashes.insert(key, value);
+        self.any_change = true;
+    }
+
+    pub fn save(&mut self) {
+        if self.any_change || !self.path.exists() {
+            let mut f = File::create(&self.path).unwrap();
+            let mut buf = String::new();
+            for (k, v) in &self.hashes {
+                buf.push_str(&format!("{}\t{}\n", k, v));
+            }
+            f.write(buf.as_ref()).unwrap();
+
+            self.any_change = false;
+        }
+    }
+}
--- a/src/main.rs
+++ b/src/main.rs
@ -1,4 +1,5 @@
 use std::env;
+use std::io;
 use std::fs;
 use std::fs::DirEntry;
 use std::fs::File;
@ -14,6 +15,11 @@ use image_utils;
 use chrono::offset::TimeZone;
 use chrono::Date;
 use chrono::Utc;
+use blake2::{Digest,Blake2b};
+
+use base64;
+
+mod hash_dict;

 #[derive(Debug)]
 struct Bread {
@ -119,11 +125,15 @@ fn main() {

    let mut channel_items = Vec::<Item>::new();

+    let mut hashes = hash_dict::HashDict::load(cwd.join(".hashes.txt")).unwrap();
+
    for bread in &breads {
        let date = bread.date.format("%Y/%m/%d").to_string();
        let date_slug = bread.date.format("%Y-%m-%d").to_string();
        let detail_file = date_slug.clone() + ".html";

+        println!("+ {}", date_slug);
+
        let (img_path, img_alt) = bread.thumb_photo();
        let note = if bread.note.is_empty() { "<i>There's no note about this bread.</i>" } else { &bread.note };

@ -133,14 +143,30 @@ fn main() {

        let image_path_encoded = utf8_percent_encode(thumb_relpath.to_str().unwrap(), DEFAULT_ENCODE_SET).to_string();

-        let im = image::open(&web_path.join(img_path)).unwrap();
-        let im = im.thumbnail(500, 500);
-        //let mut output = File::create(&thumb_path);
-        im.save(&thumb_path).unwrap();
+        // TODO keep the original path in bread so we dont have to reconstruct it here
+        let image_real_path = web_path.join(img_path);
+
+        // Create the thumb
+        {
+            let mut img_file = fs::File::open(&image_real_path).unwrap();
+            let mut hasher = Blake2b::new();
+            io::copy(&mut img_file, &mut hasher).unwrap();
+            let hash = base64::encode(&hasher.result());

-        //image_utils::resize(&web_path.join(img_path), 500, 500, &thumb_path).unwrap();
+            let hash_key = thumb_path.to_str().unwrap();
+            let old_hash = hashes.get(hash_key);
+            if old_hash.is_none() || !old_hash.unwrap().eq(&hash) {
+                println!("building thumb...");

-        // bread pic for the thumbnails page
+                let im = image::open(&image_real_path).unwrap();
+                let im = im.thumbnail(500, 500);
+                im.save(&thumb_path).unwrap();
+
+                hashes.put(hash_key.to_string(), hash);
+            }
+        }
+
+        // Prepare the thumb card for the gallery page
        {
            let thumb = thumb_tpl
                .replace("{detail_url}", &detail_file)
@ -151,7 +177,7 @@ fn main() {
            thumbs.push_str(&thumb);
        }

-        // add to rss
+        // Add to RSS
        {
            let image_url : String = channel.link().to_string() + "/" + &image_path_encoded;

@ -172,7 +198,7 @@ fn main() {
                .build().unwrap());
        }

-        // generate the detail page
+        // Generate the detail page
        {
            let detail = detail_tpl
                .replace("{head}", &head_tpl.replace("{title}", &format!("Bread from {}", date)).trim())
@ -191,15 +217,19 @@ fn main() {
        }
    }

-    let main = main_tpl.replace("{breads}", &thumbs.trim())
-                       .replace("{head}", &head_tpl.replace("{title}", "Piggo's breads").trim());
+    hashes.save();

    {
+        println!("Building the gallery page");
+        let main = main_tpl.replace("{breads}", &thumbs.trim())
+            .replace("{head}", &head_tpl.replace("{title}", "Piggo's breads").trim());
+
        let mut f = OpenOptions::new().write(true).truncate(true).create(true).open(web_path.join("index.html")).unwrap();
        f.write(main.as_bytes()).unwrap();
    }

    {
+        println!("Generating feed...");
        let f = OpenOptions::new().write(true).truncate(true).create(true).open(web_path.join("feed.xml")).unwrap();
        channel.set_items(channel_items);
        channel.pretty_write_to(f, b' ', 2).unwrap();