From 9c9647ded669057493186245f5dcbc3b248d2783 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Sun, 28 Jun 2020 16:55:54 +0200 Subject: [PATCH] web: speed up loading Tera templates --- Cargo.lock | 7 ++++--- Cargo.toml | 11 +++++----- src/web/page/templates.rs | 42 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 51 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ff4dcb28f..c572ad131 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -409,11 +409,12 @@ dependencies = [ "structopt 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", "systemstat 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "tera 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tera 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)", "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", "toml 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", "url 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "walkdir 2.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "zstd 0.5.2+zstd.1.4.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -3241,7 +3242,7 @@ dependencies = [ [[package]] name = "tera" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "chrono 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4379,7 +4380,7 @@ dependencies = [ "checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" "checksum tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" "checksum tendril 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "707feda9f2582d5d680d733e38755547a3e8fb471e7ba11452ecfd9ce93a5d3b" -"checksum tera 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "44567278e3f16c6f888f4a1426d1af33827e6bffbe3911fe24aec2c594f0dfcb" +"checksum tera 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "55df25c7768a0fb9f165931366eb0f21587c407061e1e69c1f5c2b495adfd9bb" "checksum termcolor 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f" "checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" "checksum thin-slice 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c" diff --git a/Cargo.toml b/Cargo.toml index ac9b2f452..6ad92f065 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,7 +57,8 @@ staticfile = { version = "0.4", features = [ "cache" ] } tempfile = "3.1.0" # Templating -tera = { version = "1.3.0", features = ["builtins"] } +tera = { version = "1.3.1", features = ["builtins"] } +walkdir = "2" # Template hot-reloading arc-swap = "0.4.6" @@ -67,6 +68,10 @@ notify = "4.0.15" chrono = { version = "0.4.11", features = ["serde"] } time = "0.1" # TODO: Remove once `iron` is removed +[dependencies.postgres] +version = "0.15" +features = ["with-chrono", "with-serde_json"] + [target.'cfg(not(windows))'.dependencies] # Process information procfs = "0.7" @@ -74,10 +79,6 @@ procfs = "0.7" [target.'cfg(windows)'.dependencies] path-slash = "0.1.1" -[dependencies.postgres] -version = "0.15" -features = ["with-chrono", "with-serde_json"] - [dev-dependencies] once_cell = "1.2.0" criterion = "0.3" diff --git a/src/web/page/templates.rs b/src/web/page/templates.rs index 3599c3d93..208de9814 100644 --- a/src/web/page/templates.rs +++ b/src/web/page/templates.rs @@ -2,14 +2,19 @@ use crate::db::Pool; use crate::error::Result; use arc_swap::ArcSwap; use chrono::{DateTime, Utc}; +use failure::ResultExt; use notify::{watcher, RecursiveMode, Watcher}; use postgres::Connection; use serde_json::Value; use std::collections::HashMap; +use std::path::PathBuf; use std::sync::{mpsc::channel, Arc}; use std::thread; use std::time::Duration; use tera::{Result as TeraResult, Tera}; +use walkdir::WalkDir; + +const TEMPLATES_DIRECTORY: &str = "tera-templates"; /// Holds all data relevant to templating #[derive(Debug)] @@ -84,7 +89,16 @@ fn load_rustc_resource_suffix(conn: &Connection) -> Result { } pub(super) fn load_templates(conn: &Connection) -> Result { - let mut tera = Tera::new("tera-templates/**/*")?; + // This uses a custom function to find the templates in the filesystem instead of Tera's + // builtin way (passing a glob expression to Tera::new), speeding up the startup of the + // application and running the tests. + // + // The problem with Tera's template loading code is, it walks all the files in the current + // directory and matches them against the provided glob expression. Unfortunately this means + // Tera will walk all the rustwide workspaces, the git repository and a bunch of other + // unrelated data, slowing down the search a lot. + let mut tera = Tera::default(); + tera.add_template_files(find_templates_in_filesystem(TEMPLATES_DIRECTORY)?)?; // This function will return any global alert, if present. ReturnValue::add_function_to( @@ -120,6 +134,32 @@ pub(super) fn load_templates(conn: &Connection) -> Result { Ok(tera) } +fn find_templates_in_filesystem(base: &str) -> Result)>> { + let root = std::fs::canonicalize(base)?; + + let mut files = Vec::new(); + for entry in WalkDir::new(&root) { + let entry = entry?; + let path = entry.path(); + + if !entry.metadata()?.is_file() { + continue; + } + + // Strip the root directory from the path and use it as the template name. + let name = path + .strip_prefix(&root) + .with_context(|_| format!("{} is not a child of {}", path.display(), root.display()))? + .to_str() + .ok_or_else(|| failure::format_err!("path {} is not UTF-8", path.display()))? + .to_string(); + + files.push((path.to_path_buf(), Some(name))); + } + + Ok(files) +} + /// Simple function that returns the pre-defined value. struct ReturnValue { name: &'static str,