selfhosted
/
lust
mirror of https://github.com/ChillFish8/lust.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Update caching

This commit is contained in:
Harrison Burt 2022-03-30 20:46:09 +01:00
parent 3601d9d515
commit d7e41cc6be
9 changed files with 484 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

351
Cargo.lock generated
View File

@ -39,7 +39,7 @@ version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"cipher",
"cpufeatures",
"opaque-debug",
@ -164,6 +164,12 @@ version = "3.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899"
[[package]]
name = "bytecount"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72feb31ffc86498dacdbd0fcebb56138e7177a8cc5cea4516031d15ae85a742e"
[[package]]
name = "bytemuck"
version = "1.8.0"
@ -188,6 +194,37 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8"
[[package]]
name = "camino"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f3132262930b0522068049f5870a856ab8affc80c70d08b6ecb785771a6fc23"
dependencies = [
"serde",
]
[[package]]
name = "cargo-platform"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cbdb825da8a5df079a43676dbe042702f1707b1109f713a01420fbb4cc71fa27"
dependencies = [
"serde",
]
[[package]]
name = "cargo_metadata"
version = "0.14.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa"
dependencies = [
"camino",
"cargo-platform",
"semver",
"serde",
"serde_json",
]
[[package]]
name = "cc"
version = "1.0.73"
@ -197,6 +234,12 @@ dependencies = [
"jobserver",
]
[[package]]
name = "cfg-if"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
[[package]]
name = "cfg-if"
version = "1.0.0"
@ -317,7 +360,7 @@ version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
]
[[package]]
@ -326,12 +369,12 @@ version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ae5588f6b3c3cb05239e90bd110f257254aecd01e4635400391aeae07497845"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"crossbeam-channel",
"crossbeam-deque",
"crossbeam-epoch",
"crossbeam-epoch 0.9.8",
"crossbeam-queue",
"crossbeam-utils",
"crossbeam-utils 0.8.8",
]
[[package]]
@ -340,8 +383,8 @@ version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53"
dependencies = [
"cfg-if",
"crossbeam-utils",
"cfg-if 1.0.0",
"crossbeam-utils 0.8.8",
]
[[package]]
@ -350,9 +393,24 @@ version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e"
dependencies = [
"cfg-if",
"crossbeam-epoch",
"crossbeam-utils",
"cfg-if 1.0.0",
"crossbeam-epoch 0.9.8",
"crossbeam-utils 0.8.8",
]
[[package]]
name = "crossbeam-epoch"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace"
dependencies = [
"autocfg",
"cfg-if 0.1.10",
"crossbeam-utils 0.7.2",
"lazy_static",
"maybe-uninit",
"memoffset 0.5.6",
"scopeguard",
]
[[package]]
@ -362,10 +420,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c"
dependencies = [
"autocfg",
"cfg-if",
"crossbeam-utils",
"cfg-if 1.0.0",
"crossbeam-utils 0.8.8",
"lazy_static",
"memoffset",
"memoffset 0.6.5",
"scopeguard",
]
@ -375,8 +433,19 @@ version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f25d8400f4a7a5778f0e4e52384a48cbd9b5c495d110786187fc750075277a2"
dependencies = [
"cfg-if",
"crossbeam-utils",
"cfg-if 1.0.0",
"crossbeam-utils 0.8.8",
]
[[package]]
name = "crossbeam-utils"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8"
dependencies = [
"autocfg",
"cfg-if 0.1.10",
"lazy_static",
]
[[package]]
@ -385,7 +454,7 @@ version = "0.8.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"lazy_static",
]
@ -501,7 +570,7 @@ version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"dirs-sys-next",
]
@ -528,7 +597,7 @@ version = "0.8.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7896dc8abb250ffdda33912550faa54c88ec8b998dec0b2c55ab224921ce11df"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
]
[[package]]
@ -543,6 +612,15 @@ dependencies = [
"syn",
]
[[package]]
name = "error-chain"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc"
dependencies = [
"version_check",
]
[[package]]
name = "exr"
version = "1.4.1"
@ -574,7 +652,7 @@ version = "1.0.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"crc32fast",
"libc",
"miniz_oxide 0.4.4",
@ -741,7 +819,7 @@ version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"js-sys",
"libc",
"wasi 0.10.2+wasi-snapshot-preview1",
@ -768,6 +846,12 @@ dependencies = [
"weezl",
]
[[package]]
name = "glob"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
[[package]]
name = "h2"
version = "0.3.12"
@ -1015,7 +1099,7 @@ version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
]
[[package]]
@ -1114,7 +1198,7 @@ version = "0.4.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6389c490849ff5bc16be905ae24bc913a9c8892e19b2341dbc175e14c341c2b8"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
]
[[package]]
@ -1134,6 +1218,7 @@ dependencies = [
"image",
"mimalloc",
"mime",
"moka",
"once_cell",
"poem",
"poem-openapi",
@ -1152,12 +1237,27 @@ dependencies = [
"webp",
]
[[package]]
name = "mach"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa"
dependencies = [
"libc",
]
[[package]]
name = "matches"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f"
[[package]]
name = "maybe-uninit"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00"
[[package]]
name = "md-5"
version = "0.9.1"
@ -1175,6 +1275,15 @@ version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
[[package]]
name = "memoffset"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa"
dependencies = [
"autocfg",
]
[[package]]
name = "memoffset"
version = "0.6.5"
@ -1241,6 +1350,28 @@ dependencies = [
"winapi",
]
[[package]]
name = "moka"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "55327ce58eed74a69fd80c75da97a08fbcbf61dbcb6917fcf8b84fa6435e70bb"
dependencies = [
"crossbeam-channel",
"crossbeam-epoch 0.8.2",
"crossbeam-utils 0.8.8",
"num_cpus",
"once_cell",
"parking_lot 0.12.0",
"quanta",
"scheduled-thread-pool",
"skeptic",
"smallvec",
"tagptr",
"thiserror",
"triomphe",
"uuid",
]
[[package]]
name = "multer"
version = "2.0.2"
@ -1375,7 +1506,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95"
dependencies = [
"bitflags",
"cfg-if",
"cfg-if 1.0.0",
"foreign-types",
"libc",
"once_cell",
@ -1410,6 +1541,17 @@ dependencies = [
"memchr",
]
[[package]]
name = "parking_lot"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
dependencies = [
"instant",
"lock_api",
"parking_lot_core 0.8.5",
]
[[package]]
name = "parking_lot"
version = "0.12.0"
@ -1417,7 +1559,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58"
dependencies = [
"lock_api",
"parking_lot_core",
"parking_lot_core 0.9.1",
]
[[package]]
name = "parking_lot_core"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216"
dependencies = [
"cfg-if 1.0.0",
"instant",
"libc",
"redox_syscall",
"smallvec",
"winapi",
]
[[package]]
@ -1426,7 +1582,7 @@ version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"libc",
"redox_syscall",
"smallvec",
@ -1526,7 +1682,7 @@ dependencies = [
"hyper",
"mime",
"multer",
"parking_lot",
"parking_lot 0.12.0",
"percent-encoding",
"pin-project-lite",
"poem-derive",
@ -1605,7 +1761,7 @@ version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8419d2b623c7c0896ff2d5d96e2cb4ede590fed28fcc34934f4c33c036e620a1"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"cpufeatures",
"opaque-debug",
"universal-hash",
@ -1660,6 +1816,33 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "pulldown-cmark"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34f197a544b0c9ab3ae46c359a7ec9cbbb5c7bf97054266fecb7ead794a181d6"
dependencies = [
"bitflags",
"memchr",
"unicase",
]
[[package]]
name = "quanta"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20afe714292d5e879d8b12740aa223c6a88f118af41870e8b6196e39a02238a8"
dependencies = [
"crossbeam-utils 0.8.8",
"libc",
"mach",
"once_cell",
"raw-cpuid",
"wasi 0.10.2+wasi-snapshot-preview1",
"web-sys",
"winapi",
]
[[package]]
name = "quote"
version = "1.0.17"
@ -1699,6 +1882,15 @@ dependencies = [
"getrandom",
]
[[package]]
name = "raw-cpuid"
version = "10.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "738bc47119e3eeccc7e94c4a506901aea5e7b4944ecd0829cbebf4af04ceda12"
dependencies = [
"bitflags",
]
[[package]]
name = "rayon"
version = "1.5.1"
@ -1719,7 +1911,7 @@ checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e"
dependencies = [
"crossbeam-channel",
"crossbeam-deque",
"crossbeam-utils",
"crossbeam-utils 0.8.8",
"lazy_static",
"num_cpus",
]
@ -1873,6 +2065,15 @@ version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f"
[[package]]
name = "same-file"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
dependencies = [
"winapi-util",
]
[[package]]
name = "schannel"
version = "0.1.19"
@ -1883,6 +2084,15 @@ dependencies = [
"winapi",
]
[[package]]
name = "scheduled-thread-pool"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7"
dependencies = [
"parking_lot 0.11.2",
]
[[package]]
name = "scoped_threadpool"
version = "0.1.9"
@ -1923,6 +2133,9 @@ name = "semver"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4a3381e03edd24287172047536f20cabde766e2cd3e65e6b00fb3af51c4f38d"
dependencies = [
"serde",
]
[[package]]
name = "serde"
@ -1985,7 +2198,7 @@ version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"cpufeatures",
"digest 0.10.3",
]
@ -1997,7 +2210,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800"
dependencies = [
"block-buffer 0.9.0",
"cfg-if",
"cfg-if 1.0.0",
"cpufeatures",
"digest 0.9.0",
"opaque-debug",
@ -2009,7 +2222,7 @@ version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"cpufeatures",
"digest 0.10.3",
]
@ -2038,6 +2251,21 @@ dependencies = [
"libc",
]
[[package]]
name = "skeptic"
version = "0.13.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16d23b015676c90a0f01c197bfdc786c20342c73a0afdda9025adb0bc42940a8"
dependencies = [
"bytecount",
"cargo_metadata",
"error-chain",
"glob",
"pulldown-cmark",
"tempfile",
"walkdir",
]
[[package]]
name = "slab"
version = "0.4.5"
@ -2081,6 +2309,12 @@ dependencies = [
"memchr",
]
[[package]]
name = "stable_deref_trait"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
[[package]]
name = "strsim"
version = "0.10.0"
@ -2126,13 +2360,19 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "tagptr"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
[[package]]
name = "tempfile"
version = "3.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"fastrand",
"libc",
"redox_syscall",
@ -2259,7 +2499,7 @@ dependencies = [
"mio",
"num_cpus",
"once_cell",
"parking_lot",
"parking_lot 0.12.0",
"pin-project-lite",
"signal-hook-registry",
"socket2",
@ -2349,7 +2589,7 @@ version = "0.1.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a1bdf54a7c28a2bbf701e1d2233f6c77f473486b94bee4f9678da5a148dca7f"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"pin-project-lite",
"tracing-attributes",
"tracing-core",
@ -2411,6 +2651,17 @@ dependencies = [
"tracing-log",
]
[[package]]
name = "triomphe"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c45e322b26410d7260e00f64234810c2f17d7ece356182af4df8f7ff07890f09"
dependencies = [
"memoffset 0.6.5",
"serde",
"stable_deref_trait",
]
[[package]]
name = "try-lock"
version = "0.2.3"
@ -2423,6 +2674,15 @@ version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
[[package]]
name = "unicase"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6"
dependencies = [
"version_check",
]
[[package]]
name = "unicode-bidi"
version = "0.3.7"
@ -2494,6 +2754,17 @@ version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
[[package]]
name = "walkdir"
version = "2.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56"
dependencies = [
"same-file",
"winapi",
"winapi-util",
]
[[package]]
name = "want"
version = "0.3.0"
@ -2522,7 +2793,7 @@ version = "0.2.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"wasm-bindgen-macro",
]
@ -2570,6 +2841,16 @@ version = "0.2.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2"
[[package]]
name = "web-sys"
version = "0.3.56"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c060b319f29dd25724f09a2ba1418f142f539b2be99fbf4d2d5a8f7330afb8eb"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "webp"
version = "0.2.0"

1
Cargo.toml
View File

@ -28,6 +28,7 @@ strum = { version = "0.24", features = ["derive"] }
rusoto_core = "0.47.0"
rusoto_s3 = "0.47.0"
moka = "0.8.0"
rayon = "1.5.1"
crc32fast = "1.3.2"
enum_dispatch = "0.3.8"

59
src/cache.rs Normal file
View File

@ -0,0 +1,59 @@
use std::ops::Deref;
use anyhow::anyhow;
use bytes::Bytes;
use once_cell::sync::OnceCell;
use crate::config::CacheConfig;
static GLOBAL_CACHE: OnceCell<Cache> = OnceCell::new();
pub fn new_cache(cfg: CacheConfig) -> anyhow::Result<Option<Cache>> {
if cfg.max_capacity.is_some() && cfg.max_images.is_some() {
return Err(anyhow!("Cache must be *either* based off of number of images or amount of memory, not both."))
} else if cfg.max_capacity.is_none() && cfg.max_images.is_none() {
return Ok(None)
}
let mut cache = moka::sync::CacheBuilder::default();
if let Some(max_items) = cfg.max_images {
cache = cache.max_capacity(max_items as u64)
}
if let Some(max_memory) = cfg.max_capacity {
cache = cache
.weigher(|k: &String, v: &Bytes| (k.len() + v.len()) as u32)
.max_capacity((max_memory * 1024 * 1024) as u64);
}
Ok(Some(cache.build().into()))
}
pub fn init_cache(cfg: CacheConfig) -> anyhow::Result<()> {
if let Some(cache) = new_cache(cfg)? {
let _ = GLOBAL_CACHE.set(cache);
};
Ok(())
}
pub fn global_cache<'a>() -> Option<&'a Cache> {
GLOBAL_CACHE.get()
}
pub struct Cache {
inner: moka::sync::Cache<String, Bytes>,
}
impl Deref for Cache {
type Target = moka::sync::Cache<String, Bytes>;
fn deref(&self) -> &Self::Target {
&self.inner
}
}
impl From<moka::sync::Cache<String, Bytes>> for Cache {
fn from(v: moka::sync::Cache<String, Bytes>) -> Self {
Self {
inner: v
}
}
}

80
src/controller.rs
View File

@ -1,10 +1,12 @@
use std::hash::Hash;
use std::sync::Arc;
use bytes::Bytes;
use once_cell::sync::OnceCell;
use uuid::Uuid;
use poem_openapi::Object;
use tokio::sync::{Semaphore, SemaphorePermit};
use tokio::time::Instant;
use crate::cache::{Cache, global_cache};
use crate::config::{BucketConfig, ImageKind};
use crate::pipelines::{PipelineController, ProcessingMode, StoreEntry};
@ -56,6 +58,7 @@ pub struct UploadInfo {
pub struct BucketController {
bucket_id: u32,
cache: Option<Cache>,
global_limiter: Option<Arc<Semaphore>>,
config: BucketConfig,
pipeline: PipelineController,
@ -66,6 +69,7 @@ pub struct BucketController {
impl BucketController {
pub fn new(
bucket_id: u32,
cache: Option<Cache>,
global_limiter: Option<Arc<Semaphore>>,
config: BucketConfig,
pipeline: PipelineController,
@ -73,6 +77,7 @@ impl BucketController {
) -> Self {
Self {
bucket_id,
cache,
global_limiter,
limiter: config.max_concurrency.map(Semaphore::new),
config,
@ -104,8 +109,18 @@ impl BucketController {
image_id,
store_entry.kind,
store_entry.sizing_id,
store_entry.data,
store_entry.data.clone(),
).await?;
if let Some(ref cache) = self.cache {
let cache_key = self.cache_key(
store_entry.sizing_id,
image_id,
store_entry.kind,
);
cache.insert(cache_key, store_entry.data);
}
}
Ok(UploadInfo {
@ -133,13 +148,12 @@ impl BucketController {
desired_kind
};
let maybe_existing = self.storage.fetch(self.bucket_id, image_id, fetch_kind, sizing_id).await?;
let maybe_existing = self.caching_fetch(image_id, fetch_kind, sizing_id).await?;
let (data, retrieved_kind) = match maybe_existing {
// If we're in JIT mode we want to re-encode the image and store it.
None => if self.config.mode == ProcessingMode::Jit {
let base_kind = self.config.formats.original_image_store_format;
let value = self.storage.fetch(
self.bucket_id,
let value = self.caching_fetch(
image_id,
base_kind,
sizing_id,
@ -192,7 +206,63 @@ impl BucketController {
pub async fn delete(&self, image_id: Uuid) -> anyhow::Result<()> {
let _permit = get_optional_permit(&self.global_limiter, &self.limiter).await?;
self.storage.delete(self.bucket_id, image_id).await
let purged_entities = self.storage.delete(self.bucket_id, image_id).await?;
if let Some(ref cache) = self.cache {
for (sizing_id, kind) in purged_entities {
let cache_key = self.cache_key(sizing_id, image_id, kind);
cache.invalidate(&cache_key);
}
}
Ok(())
}
}
impl BucketController {
#[inline]
fn cache_key(&self, sizing_id: u32, image_id: Uuid, kind: ImageKind) -> String {
format!(
"{bucket}:{sizing}:{image}:{kind}",
bucket = self.bucket_id,
sizing = sizing_id,
image = image_id,
kind = kind.as_file_extension(),
)
}
async fn caching_fetch(
&self,
image_id: Uuid,
fetch_kind: ImageKind,
sizing_id: u32,
) -> anyhow::Result<Option<Bytes>> {
let maybe_cache_backend = self.cache
.as_ref()
.map(Some)
.unwrap_or_else(global_cache);
let cache_key = self.cache_key(sizing_id, image_id, fetch_kind);
if let Some(cache) = maybe_cache_backend {
if let Some(buffer) = cache.get(&cache_key) {
return Ok(Some(buffer))
}
}
let maybe_existing = self.storage.fetch(
self.bucket_id,
image_id,
fetch_kind,
sizing_id
).await?;
if let Some(cache) = maybe_cache_backend {
if let Some(ref buffer) = maybe_existing {
cache.insert(cache_key, buffer.clone());
}
}
Ok(maybe_existing)
}
}

15
src/main.rs
View File

@ -8,6 +8,7 @@ mod processor;
#[cfg(test)]
mod tests;
mod cache;
use std::path::PathBuf;
use std::sync::Arc;
@ -71,6 +72,10 @@ async fn main() -> Result<()> {
config::init(&args.config_file).await?;
if let Some(config) = config::config().global_cache {
cache::init_cache(config)?;
}
let global_limiter = config::config()
.max_concurrency
.map(Semaphore::new)
@ -87,16 +92,22 @@ async fn main() -> Result<()> {
.map(|(bucket, cfg)| {
let bucket_id = crate::utils::crc_hash(bucket);
let pipeline = cfg.mode.build_pipeline(cfg);
let cache = cfg.cache
.map(cache::new_cache)
.transpose()?
.flatten();
let controller = BucketController::new(
bucket_id,
cache,
global_limiter.clone(),
cfg.clone(),
pipeline,
storage.clone(),
);
(bucket_id, controller)
Ok::<_, anyhow::Error>((bucket_id, controller))
})
.collect();
.collect::<Result<hashbrown::HashMap<_, _>, anyhow::Error>>()?;
controller::init_buckets(buckets);

6
src/storage/backends/blob_storage.rs
View File

@ -130,11 +130,12 @@ impl StorageBackend for BlobStorageBackend {
&self,
bucket_id: u32,
image_id: Uuid,
) -> anyhow::Result<()> {
) -> anyhow::Result<Vec<(u32, ImageKind)>> {
let bucket = get_bucket_by_id(bucket_id)
.ok_or_else(|| anyhow!("Bucket does not exist."))?
.cfg();
let mut hit_entries = vec![];
for sizing_id in bucket.sizing_preset_ids().iter().copied() {
for kind in ImageKind::variants() {
let store_in = self.format_path(bucket_id, sizing_id, image_id, *kind);
@ -146,10 +147,11 @@ impl StorageBackend for BlobStorageBackend {
..Default::default()
};
self.client.delete_object(request).await?;
hit_entries.push((sizing_id, *kind));
}
}
Ok(())
Ok(hit_entries)
}
}

9
src/storage/backends/filesystem.rs
View File

@ -75,11 +75,12 @@ impl StorageBackend for FileSystemBackend {
&self,
bucket_id: u32,
image_id: Uuid,
) -> anyhow::Result<()> {
) -> anyhow::Result<Vec<(u32, ImageKind)>> {
let bucket = get_bucket_by_id(bucket_id)
.ok_or_else(|| anyhow!("Bucket does not exist."))?
.cfg();
let mut hit_entries = vec![];
for sizing_id in bucket.sizing_preset_ids().iter().copied() {
for kind in ImageKind::variants() {
let store_in = self.format_path(bucket_id, sizing_id);
@ -87,14 +88,16 @@ impl StorageBackend for FileSystemBackend {
debug!("Purging image @ {:?}", &path);
match tokio::fs::remove_file(&path).await {
Ok(()) => continue,
Ok(()) => {
hit_entries.push((sizing_id, *kind));
},
Err(ref e) if e.kind() == ErrorKind::NotFound => continue,
Err(other) => return Err(other.into()),
}
}
}
Ok(())
Ok(hit_entries)
}
}

2
src/storage/template.rs
View File

@ -26,5 +26,5 @@ pub trait StorageBackend: Sync + Send + 'static {
&self,
bucket_id: u32,
image_id: Uuid,
) -> anyhow::Result<()>;
) -> anyhow::Result<Vec<(u32, ImageKind)>>;
}

12
src/tests.rs
View File

@ -6,7 +6,7 @@ use poem::test::TestClient;
use poem::web::headers;
use tokio::sync::Semaphore;
use crate::{BucketController, config, controller, StorageBackend};
use crate::{BucketController, cache, config, controller, StorageBackend};
const JIT_CONFIG: &str = include_str!("../tests/configs/jit-mode.yaml");
const AOT_CONFIG: &str = include_str!("../tests/configs/aot-mode.yaml");
@ -32,16 +32,22 @@ async fn setup_environment(cfg: &str) -> anyhow::Result<TestClient<Route>> {
.map(|(bucket, cfg)| {
let bucket_id = crate::utils::crc_hash(bucket);
let pipeline = cfg.mode.build_pipeline(cfg);
let cache = cfg.cache
.map(cache::new_cache)
.transpose()?
.flatten();
let controller = BucketController::new(
bucket_id,
cache,
global_limiter.clone(),
cfg.clone(),
pipeline,
storage.clone(),
);
(bucket_id, controller)
Ok::<_, anyhow::Error>((bucket_id, controller))
})
.collect();
.collect::<Result<hashbrown::HashMap<_, _>, anyhow::Error>>()?;
controller::init_buckets(buckets);