commit 24294ad837941339890feeb28b42c5eab5d115ec from: mtmn date: Tue Jul 28 17:17:18 2026 UTC migrate from sled to lmdb commit - c1b439fd702cbfcbea3cc8fb5fbdfa40383410e1 commit + 24294ad837941339890feeb28b42c5eab5d115ec blob - 9fdee92b2fdc5fbc43e688b399f59474365caaf2 blob + 3fa3998102554b6bd678011e043cd5702f9a273d --- Cargo.lock +++ Cargo.lock @@ -89,19 +89,13 @@ source = "registry+https://github.com/rust-lang/crates checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] -name = "bincode" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ - "serde", -] - -[[package]] name = "bitflags" version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +dependencies = [ + "serde_core", +] [[package]] name = "bumpalo" @@ -116,23 +110,12 @@ source = "registry+https://github.com/rust-lang/crates checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] -name = "cache-advisor" -version = "1.0.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f89ab55ca4e6a46a0740a1c5346db1ad66e4a76598bbfa060dc3259935a7450" -dependencies = [ - "crossbeam-queue", -] - -[[package]] name = "cc" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", - "jobserver", - "libc", "shlex", ] @@ -196,17 +179,6 @@ source = "registry+https://github.com/rust-lang/crates checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] -name = "concurrent-map" -version = "5.0.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6542c565fbcba786db59307d7840f0bf5cd9e0aba6502755337e15f0e06fd65" -dependencies = [ - "ebr", - "serde", - "stack-map", -] - -[[package]] name = "convert_case" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -354,15 +326,26 @@ dependencies = [ ] [[package]] -name = "ebr" -version = "0.2.13" +name = "displaydoc" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b1ea3b18359d566f360eaf811a2d69bc6c8eb6faaeecc8839975633860a076e" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ - "shared-local-state", + "proc-macro2", + "quote", + "syn 3.0.3", ] [[package]] +name = "doxygen-rs" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "415b6ec780d34dcf624666747194393603d0373b7141eef01d12ee58881507d9" +dependencies = [ + "phf", +] + +[[package]] name = "either" version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -399,7 +382,7 @@ dependencies = [ [[package]] name = "essentia" version = "0.1.4" -source = "git+ssh://anonymous@mtmn.name/essentia-rs.git#9470b195fe6e4d522289221c02e066115bdf745d" +source = "git+https://github.com/lagmoellertim/essentia-rs.git#9470b195fe6e4d522289221c02e066115bdf745d" dependencies = [ "cxx", "essentia-codegen", @@ -410,7 +393,7 @@ dependencies = [ [[package]] name = "essentia-codegen" version = "0.1.4" -source = "git+ssh://anonymous@mtmn.name/essentia-rs.git#9470b195fe6e4d522289221c02e066115bdf745d" +source = "git+https://github.com/lagmoellertim/essentia-rs.git#9470b195fe6e4d522289221c02e066115bdf745d" dependencies = [ "convert_case", "essentia-core", @@ -425,7 +408,7 @@ dependencies = [ [[package]] name = "essentia-core" version = "0.1.4" -source = "git+ssh://anonymous@mtmn.name/essentia-rs.git#9470b195fe6e4d522289221c02e066115bdf745d" +source = "git+https://github.com/lagmoellertim/essentia-rs.git#9470b195fe6e4d522289221c02e066115bdf745d" dependencies = [ "cxx", "essentia-sys", @@ -438,7 +421,7 @@ dependencies = [ [[package]] name = "essentia-sys" version = "0.1.4" -source = "git+ssh://anonymous@mtmn.name/essentia-rs.git#9470b195fe6e4d522289221c02e066115bdf745d" +source = "git+https://github.com/lagmoellertim/essentia-rs.git#9470b195fe6e4d522289221c02e066115bdf745d" dependencies = [ "cxx", "cxx-build", @@ -464,12 +447,6 @@ source = "registry+https://github.com/rust-lang/crates checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] -name = "fault-injection" -version = "1.0.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e3d175246dec3fddef3b1fcd57acdb023e4c562d032e9eccc5f246da3d7fed3" - -[[package]] name = "find-msvc-tools" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -486,25 +463,18 @@ dependencies = [ ] [[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] name = "foldhash" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" [[package]] -name = "fs2" -version = "0.4.3" +name = "form_urlencoded" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" dependencies = [ - "libc", - "winapi", + "percent-encoding", ] [[package]] @@ -580,6 +550,143 @@ dependencies = [ ] [[package]] +name = "heed" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad82d6598ccf1dac15c8b758a1bd282b755b6776be600429176757190a1b0202" +dependencies = [ + "bitflags", + "byteorder", + "heed-traits", + "heed-types", + "libc", + "lmdb-master-sys", + "once_cell", + "page_size", + "synchronoise", + "url", +] + +[[package]] +name = "heed-traits" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb3130048d404c57ce5a1ac61a903696e8fcde7e8c2991e9fcfc1f27c3ef74ff" + +[[package]] +name = "heed-types" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c255bdf46e07fb840d120a36dcc81f385140d7191c76a7391672675c01a55d" +dependencies = [ + "byteorder", + "heed-traits", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] name = "indexmap" version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -590,32 +697,12 @@ dependencies = [ ] [[package]] -name = "inline-array" -version = "0.1.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45e8b42f7d66073247744b2971fcc4df24afe3e686616c20a98439ec4f156d43" -dependencies = [ - "concurrent-map", - "serde", -] - -[[package]] name = "is_terminal_polyfill" version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" [[package]] -name = "jobserver" -version = "0.1.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" -dependencies = [ - "getrandom 0.4.3", - "libc", -] - -[[package]] name = "js-sys" version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -666,6 +753,23 @@ source = "registry+https://github.com/rust-lang/crates checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lmdb-master-sys" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaeb9bd22e73bd1babffff614994b341e9b2008de7bb73bf1f7e9154f1978f8b" +dependencies = [ + "cc", + "doxygen-rs", + "libc", +] + +[[package]] name = "lock_api" version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -847,32 +951,13 @@ source = "registry+https://github.com/rust-lang/crates checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" [[package]] -name = "pagetable" -version = "0.4.6" +name = "page_size" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69b57830c885fc057ecbf2f1f99f0427c3d102cf2ee5e80a52c09948d45a460e" - -[[package]] -name = "parking_lot" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" -dependencies = [ - "cfg-if", "libc", - "redox_syscall", - "smallvec", - "windows-link", + "winapi", ] [[package]] @@ -882,6 +967,54 @@ source = "registry+https://github.com/rust-lang/crates checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] name = "pkg-config" version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -916,6 +1049,15 @@ dependencies = [ ] [[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] name = "prettyplease" version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -950,6 +1092,21 @@ source = "registry+https://github.com/rust-lang/crates checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" + +[[package]] name = "rawpointer" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -976,15 +1133,6 @@ dependencies = [ ] [[package]] -name = "redox_syscall" -version = "0.5.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" -dependencies = [ - "bitflags", -] - -[[package]] name = "redox_users" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1135,15 +1283,6 @@ dependencies = [ ] [[package]] -name = "shared-local-state" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a50ccb2f45251772ed15abfd1e5f10a305288187b1582ab2e4295b29bbb4929" -dependencies = [ - "parking_lot", -] - -[[package]] name = "shlex" version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1156,28 +1295,10 @@ source = "registry+https://github.com/rust-lang/crates checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" [[package]] -name = "sled" -version = "1.0.0-alpha.124" -source = "git+https://github.com/spacejam/sled?rev=e449d171#e449d17111f4a097e1c66b6db241962ccb6a4136" -dependencies = [ - "bincode", - "cache-advisor", - "concurrent-map", - "crc32fast", - "crossbeam-queue", - "ebr", - "fault-injection", - "fnv", - "fs2", - "inline-array", - "log", - "pagetable", - "parking_lot", - "rayon", - "serde", - "stack-map", - "zstd", -] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" [[package]] name = "smallvec" @@ -1219,15 +1340,6 @@ source = "registry+https://github.com/rust-lang/crates checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] -name = "stack-map" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49d6d36fee60faad91e23603db2356677b58ec2429237b39d5c60c26868f37c" -dependencies = [ - "serde", -] - -[[package]] name = "strsim" version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1256,6 +1368,26 @@ dependencies = [ ] [[package]] +name = "synchronoise" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dbc01390fc626ce8d1cffe3376ded2b72a11bb70e1c75f404a210e4daa4def2" +dependencies = [ + "crossbeam-queue", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] name = "tempfile" version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1309,6 +1441,16 @@ dependencies = [ ] [[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1333,6 +1475,24 @@ source = "registry+https://github.com/rust-lang/crates checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" [[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] name = "utf8parse" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1352,13 +1512,13 @@ dependencies = [ "clap", "dirs", "essentia", + "heed", "lofty", "postcard", "rayon", "regex", "rusqlite", "serde", - "sled", "tempfile", "walkdir", ] @@ -1471,30 +1631,84 @@ dependencies = [ ] [[package]] -name = "zstd" -version = "0.12.4" +name = "writeable" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" dependencies = [ - "zstd-safe", + "stable_deref_trait", + "yoke-derive", + "zerofrom", ] [[package]] -name = "zstd-safe" -version = "6.0.6" +name = "yoke-derive" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ - "libc", - "zstd-sys", + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", ] [[package]] -name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" +name = "zerofrom" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" dependencies = [ - "cc", - "pkg-config", + "zerofrom-derive", ] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] blob - ce80bf5fe012ab443df6db1c60207ecefcbf7504 blob + f013dc111bb52ca8467ef56f689bf4fe9512a580 --- Cargo.toml +++ Cargo.toml @@ -11,11 +11,12 @@ lofty = "0.24.0" rayon = "1.10" serde = { version = "1.0", features = ["derive"] } postcard = { version = "1", features = ["alloc"] } -sled = { git = "https://github.com/spacejam/sled", rev = "e449d171" } +# Records go through a postcard codec, so heed's serde codecs are dead weight. +heed = { version = "0.22", default-features = false, features = ["longer-keys"] } dirs = "6" regex = "1" rusqlite = "0.40" -essentia = { git = "ssh://anonymous@mtmn.name/essentia-rs.git" } +essentia = { git = "https://github.com/lagmoellertim/essentia-rs.git" } [dev-dependencies] tempfile = "3" blob - fc922f8701396116387c17d046553d3473839865 blob + 464b0361b9d4cf54e96240dc2bc1576b54cecfb8 --- README.md +++ README.md @@ -83,6 +83,10 @@ virittaa [OPTIONS] [PATH]... Analysis saves to the database unless `--no-store`, and skips files that already have both tags unless `--force`. +Reports are stored in LMDB, in `data.mdb` under the database directory. A +directory left by a sled-era release is not read; those entries have to be +re-analyzed or re-imported. + ```sh virittaa -tf ~/music/ # analyze, overwrite existing tags virittaa -j 4 --no-store track.mp3 # 4 threads, nothing stored blob - e1ef264c02d9944b86913351622319314bb4b04c blob + 25e4bfa043bb9b59b47be1814d894b2b6374eb9a --- flake.lock +++ flake.lock @@ -2,11 +2,11 @@ "nodes": { "crane": { "locked": { - "lastModified": 1780532242, - "narHash": "sha256-D+BsdpxmtUwtqGoY0IXPhHgTlmqgcZKCEo1oMyn7ep0=", + "lastModified": 1784564969, + "narHash": "sha256-TkTWNhJV/8Wk3czlbz4bwHZkFCaCHPsOy+oJe4PUiXg=", "owner": "ipetkov", "repo": "crane", - "rev": "59a82a1222dd3b2080b5cc52a1a2e8d5f1b77f37", + "rev": "7930f6c291de6f83c257839d434592aa085f290a", "type": "github" }, "original": { @@ -20,15 +20,15 @@ "locked": { "lastModified": 1751989263, "narHash": "sha256-LCiqug2FM6gTP3HY5YxT7CSd5CiU90g0hUCDhv3GNTk=", - "ref": "refs/heads/main", + "owner": "lagmoellertim", + "repo": "essentia-rs", "rev": "9470b195fe6e4d522289221c02e066115bdf745d", - "revCount": 58, - "type": "git", - "url": "ssh://anonymous@mtmn.name/essentia-rs.git" + "type": "github" }, "original": { - "type": "git", - "url": "ssh://anonymous@mtmn.name/essentia-rs.git" + "owner": "lagmoellertim", + "repo": "essentia-rs", + "type": "github" } }, "fenix": { @@ -39,11 +39,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1781430890, - "narHash": "sha256-Dict8ZPqm3fei8iFXh75sPBNPqCdn7a8GJytgLajbGw=", + "lastModified": 1785146392, + "narHash": "sha256-XOU4aG8jroBwvA4vegmD7m7jopuRHJIMX3j7/WUUJgM=", "owner": "nix-community", "repo": "fenix", - "rev": "1b9586dd32bb28519cf6eea855e2df5b4d5eca15", + "rev": "44a90ec0ed3ba33823fa5acd6d8614b7d3eaa9b1", "type": "github" }, "original": { @@ -72,11 +72,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1781074563, - "narHash": "sha256-md8WlXOlfnIeHeOScMTTHFyf2d6iaTwPl2apR5EQ3P4=", + "lastModified": 1785090369, + "narHash": "sha256-m0pDuRJG7EDo9ri+4Ksu83VsI+PlxNC9lNBfydejce4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "9ae611a455b90cf061d8f332b977e387bda8e1ca", + "rev": "624af665418d3c65d544145b4d34ad696439570e", "type": "github" }, "original": { @@ -97,11 +97,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1781382341, - "narHash": "sha256-1AuqkRHiBSuU8AjzPW4gFgwyPhFaMZhYJVtMormLvb8=", + "lastModified": 1785084284, + "narHash": "sha256-aa+pBL1RU/8E/H4AXsogUI3pDukwsqb+uGl9Pwys/B0=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "5017a50153285a2d9bfa286e6fa96723e997d5b6", + "rev": "12c3381f0b17b8eec21075d1c72fd010996a9bda", "type": "github" }, "original": { blob - 4d3c9b3c15e836787259a020586c378f460e2109 blob + 92ced667a2e3421ae3fc703428ebf3c0ee62d9c9 --- flake.nix +++ flake.nix @@ -11,7 +11,7 @@ crane.url = "github:ipetkov/crane"; essentia-rs = { - url = "git+ssh://anonymous@mtmn.name/essentia-rs.git"; + url = "github:lagmoellertim/essentia-rs"; flake = false; }; }; @@ -46,7 +46,7 @@ ] ++ pkgs.lib.optionals (!pkgs.stdenv.isDarwin) [pkgs.stdenv.cc.cc.lib]; - essentiaRsUrl = "ssh://anonymous@mtmn.name/essentia-rs.git"; + essentiaRsUrl = "https://github.com/lagmoellertim/essentia-rs.git"; src = pkgs.runCommandLocal "virittaa-src" {} '' cp -r ${craneLib.cleanCargoSource ./.} $out blob - 54aacdf2f6fcd019df2da57d3db7c3be64289122 blob + 3a09078370a999e07ec67b8293a33a7f3f52722e --- hack/void-build.rb +++ hack/void-build.rb @@ -26,7 +26,7 @@ VOID_REPO = "https://repo-default.voidlinux.org/curren ESSENTIA_REV = "b9fa6cb674ca43dfb94d28d293aeda441c6745db" # The git dependency in Cargo.toml; cloned unless -e gives a local checkout. -ESSENTIA_RS_URL = "ssh://anonymous@mtmn.name/essentia-rs.git" +ESSENTIA_RS_URL = "https://github.com/lagmoellertim/essentia-rs.git" # ffmpeg6-devel rather than ffmpeg-devel, which is still 4.4.x; either # provides libswresample, but essentia and virittaa must link the same one. blob - 137726ca6b40f7b5702c2637008f55d5e75ce63a blob + 492fff9ccd647fa6975a6a6dedbf9f61ca53ccf5 --- src/audio_processing.rs +++ src/audio_processing.rs @@ -33,7 +33,7 @@ pub const WORKER_ARG: &str = "analyze-one"; static ESSENTIA: LazyLock = LazyLock::new(Essentia::new); /// Result of analysing a single track. -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct Analysis { /// Estimated tempo in beats per minute. pub bpm: f32, blob - 8be736f201e7b934f721c5579ceb1d9e3e82a710 blob + c34c0c8ed39654c11b201da10018f59e1e5e7d26 --- src/db.rs +++ src/db.rs @@ -1,9 +1,12 @@ use anyhow::{Context, Result}; +use heed::types::{Bytes, Str}; +use heed::{BoxedError, BytesDecode, BytesEncode, Database, Env, EnvOpenOptions, RoTxn, RwTxn}; use regex::Regex; use serde::{Deserialize, Serialize}; -use sled::Tree; +use std::borrow::Cow; use std::cmp::Reverse; use std::fmt; +use std::ops::Bound; use std::path::Path; #[derive(Debug, Serialize, Deserialize)] @@ -70,26 +73,55 @@ impl fmt::Display for TrackReport { } } -/// Sled-backed library database with secondary indexes for artist, key, and BPM. +/// postcard codec for `TrackReport`, carrying the legacy fallback into heed. +pub struct ReportCodec; + +impl<'a> BytesEncode<'a> for ReportCodec { + type EItem = TrackReport; + + fn bytes_encode(item: &'a Self::EItem) -> Result, BoxedError> { + serialize_report(item).map(Cow::Owned).map_err(Into::into) + } +} + +impl<'a> BytesDecode<'a> for ReportCodec { + type DItem = TrackReport; + + fn bytes_decode(bytes: &'a [u8]) -> Result { + deserialize_report(bytes).map_err(Into::into) + } +} + +/// LMDB-backed library database with secondary indexes for artist, key, and BPM. /// -/// Schema: -/// - default tree: `"Artist - Track"` → postcard-encoded `TrackReport` -/// - `"artist"` tree: `"artist\0primary_key"` → 8-byte BE timestamp -/// - `"key"` tree: `"key\0primary_key"` → 8-byte BE timestamp -/// - `"bpm"` tree: `f64_be_bytes + "\0" + primary_key` → 8-byte BE timestamp -/// - `"time"` tree: `i64_be_bytes(timestamp) + "\0" + primary_key` → empty +/// Schema, one named database each: +/// - `tracks`: `"Artist - Track"` → postcard-encoded `TrackReport` +/// - `artist`: `"artist\0primary_key"` → 8-byte BE timestamp +/// - `key`: `"key\0primary_key"` → 8-byte BE timestamp +/// - `bpm`: `f64_be_bytes + "\0" + primary_key` → 8-byte BE timestamp +/// - `time`: `i64_be_bytes(timestamp) + "\0" + primary_key` → empty pub struct LibraryDb { - db: sled::Db, - artist_tree: Tree, - key_tree: Tree, - bpm_tree: Tree, - time_tree: Tree, + env: Env, + tracks: Database, + artist_db: Database, + key_db: Database, + bpm_db: Database, + time_db: Database, } const SEP: u8 = 0x00; const BPM_KEY_LEN: usize = 8; // f64 big-endian const TIME_KEY_LEN: usize = 8; // i64 big-endian +/// Address space LMDB reserves up front; the file grows into it as needed. +/// A 159k-track library measured 535 bytes each across the record and its four +/// index entries, so this holds roughly 8 million. Exceeding it is a hard +/// error, and reserving address space on 64-bit costs nothing. +const MAP_SIZE: usize = 4 << 30; + +/// `tracks` plus the four secondary indexes. +const MAX_DBS: u32 = 5; + fn prefix_index_key(value: &[u8], primary_key: &[u8]) -> Vec { let mut buf = Vec::with_capacity(value.len() + 1 + primary_key.len()); buf.extend_from_slice(value); @@ -162,143 +194,139 @@ impl LibraryDb { /// /// Returns an error if the database cannot be opened or created. pub fn open(path: &Path) -> Result { - let db = sled::open(path) - .with_context(|| format!("Failed to open database at {}", path.display()))?; - let artist_tree = db - .open_tree("artist") - .context("Failed to open artist tree")?; - let key_tree = db.open_tree("key").context("Failed to open key tree")?; - let bpm_tree = db.open_tree("bpm").context("Failed to open bpm tree")?; - let time_tree = db.open_tree("time").context("Failed to open time tree")?; + // LMDB expects the directory to be there already. + std::fs::create_dir_all(path) + .with_context(|| format!("Failed to create database directory {}", path.display()))?; + + // SAFETY: LMDB memory-maps the data file, so nothing else may write to + // it while the environment is open. virittaa only reaches it from here. + let env = unsafe { + EnvOpenOptions::new() + .map_size(MAP_SIZE) + .max_dbs(MAX_DBS) + .open(path) + } + .with_context(|| format!("Failed to open database at {}", path.display()))?; + + let mut wtxn = env + .write_txn() + .context("Failed to begin a write transaction")?; + let tracks = env + .create_database(&mut wtxn, Some("tracks")) + .context("Failed to open tracks database")?; + let artist_db = env + .create_database(&mut wtxn, Some("artist")) + .context("Failed to open artist index")?; + let key_db = env + .create_database(&mut wtxn, Some("key")) + .context("Failed to open key index")?; + let bpm_db = env + .create_database(&mut wtxn, Some("bpm")) + .context("Failed to open bpm index")?; + let time_db = env + .create_database(&mut wtxn, Some("time")) + .context("Failed to open time index")?; + wtxn.commit().context("Failed to create databases")?; + Ok(Self { - db, - artist_tree, - key_tree, - bpm_tree, - time_tree, + env, + tracks, + artist_db, + key_db, + bpm_db, + time_db, }) } /// Write secondary index entries for a report. - fn update_indexes(&self, key: &str, report: &TrackReport) -> Result<()> { + fn update_indexes(&self, wtxn: &mut RwTxn, key: &str, report: &TrackReport) -> Result<()> { let ts = report.timestamp.unwrap_or(0); let ts_bytes = ts.to_be_bytes(); // Artist index let artist_lower = report.artist.as_deref().unwrap_or("unknown").to_lowercase(); let idx_key = prefix_index_key(artist_lower.as_bytes(), key.as_bytes()); - self.artist_tree.insert(&idx_key, &ts_bytes[..])?; + self.artist_db.put(wtxn, &idx_key, &ts_bytes[..])?; // Key-signature index let key_lower = report.key.as_deref().unwrap_or("unknown").to_lowercase(); let idx_key = prefix_index_key(key_lower.as_bytes(), key.as_bytes()); - self.key_tree.insert(&idx_key, &ts_bytes[..])?; + self.key_db.put(wtxn, &idx_key, &ts_bytes[..])?; // BPM index if let Some(bpm) = report.bpm { let idx_key = bpm_index_key(bpm, key.as_bytes()); - self.bpm_tree.insert(&idx_key, &ts_bytes[..])?; + self.bpm_db.put(wtxn, &idx_key, &ts_bytes[..])?; } // Time index (always written, timestamp=0 for entries without one) let idx_key = time_index_key(ts, key.as_bytes()); - self.time_tree.insert(&idx_key, &[][..])?; + self.time_db.put(wtxn, &idx_key, &[])?; Ok(()) } /// Remove all secondary index entries for a report. - fn remove_indexes(&self, key: &str, report: &TrackReport) -> Result<()> { + fn remove_indexes(&self, wtxn: &mut RwTxn, key: &str, report: &TrackReport) -> Result<()> { let artist_lower = report.artist.as_deref().unwrap_or("unknown").to_lowercase(); let idx_key = prefix_index_key(artist_lower.as_bytes(), key.as_bytes()); - self.artist_tree.remove(&idx_key)?; + self.artist_db.delete(wtxn, &idx_key)?; let key_lower = report.key.as_deref().unwrap_or("unknown").to_lowercase(); let idx_key = prefix_index_key(key_lower.as_bytes(), key.as_bytes()); - self.key_tree.remove(&idx_key)?; + self.key_db.delete(wtxn, &idx_key)?; if let Some(bpm) = report.bpm { let idx_key = bpm_index_key(bpm, key.as_bytes()); - self.bpm_tree.remove(&idx_key)?; + self.bpm_db.delete(wtxn, &idx_key)?; } let ts = report.timestamp.unwrap_or(0); let idx_key = time_index_key(ts, key.as_bytes()); - self.time_tree.remove(&idx_key)?; + self.time_db.delete(wtxn, &idx_key)?; Ok(()) } + /// Insert or replace one entry and its index entries within `wtxn`. + fn write_entry(&self, wtxn: &mut RwTxn, key: &str, report: &TrackReport) -> Result<()> { + // Drop stale indexes if this overwrites an existing entry. + if let Some(old) = self.tracks.get(wtxn, key)? { + self.remove_indexes(wtxn, key, &old)?; + } + self.tracks.put(wtxn, key, report)?; + self.update_indexes(wtxn, key, report) + } + /// Get a track report by exact primary key. /// /// # Errors /// /// Returns an error if the database operation fails. pub fn get(&self, key: &str) -> Result> { - self.db - .get(key.as_bytes())? - .map(|bytes| deserialize_report(&bytes)) - .transpose() + let rtxn = self.env.read_txn()?; + self.tracks + .get(&rtxn, key) + .context("Failed to read track report") } - /// Save a single track report, inserting or updating as needed. - /// - /// # Errors - /// - /// Returns an error if the database operation fails. - pub fn save(&self, key: &str, report: &TrackReport) -> Result<()> { - // Remove stale indexes if overwriting an existing entry. - if let Some(old_bytes) = self - .db - .insert(key.as_bytes(), serialize_report(report)?.as_slice())? - { - let old_report = deserialize_report(&old_bytes)?; - self.remove_indexes(key, &old_report)?; - } - - self.update_indexes(key, report)?; - Ok(()) - } - /// Save a batch of track reports, inserting or updating as needed. /// - /// Uses sled's batch API for the default tree to reduce write amplification. + /// The whole batch, records and indexes alike, is one transaction: a failure + /// part way through leaves the database exactly as it was. /// /// # Errors /// /// Returns an error if the database operation fails. pub fn save_batch(&self, entries: &[(String, TrackReport)]) -> Result<()> { + let mut wtxn = self.env.write_txn()?; for (key, report) in entries { - // Remove stale indexes for any existing entries. - if let Some(old_bytes) = self.db.get(key.as_bytes())? { - let old_report = deserialize_report(&old_bytes)?; - self.remove_indexes(key, &old_report)?; - } - self.update_indexes(key, report)?; + self.write_entry(&mut wtxn, key, report)?; } - - let mut batch = sled::Batch::default(); - for (key, report) in entries { - let serialized = serialize_report(report)?; - batch.insert(key.as_bytes(), serialized.as_slice()); - } - self.db.apply_batch(batch)?; + wtxn.commit().context("Failed to commit track reports")?; Ok(()) } - /// Delete a track report and its secondary index entries. - /// - /// # Errors - /// - /// Returns an error if the database operation fails. - pub fn delete(&self, key: &str) -> Result<()> { - if let Some(old_bytes) = self.db.remove(key.as_bytes())? { - let old_report = deserialize_report(&old_bytes)?; - self.remove_indexes(key, &old_report)?; - } - Ok(()) - } - /// List up to `limit` track reports (pass `0` for no limit), /// ordered chronologically (oldest first, most recent at the bottom). /// @@ -306,9 +334,10 @@ impl LibraryDb { /// /// Returns an error if the database operation fails. pub fn list(&self, limit: usize) -> Result> { + let rtxn = self.env.read_txn()?; let mut candidates: Vec<(String, i64)> = Vec::new(); - for item in &self.time_tree { + for item in self.time_db.iter(&rtxn)? { let (idx_key, _) = item.context("Failed to read time index")?; let primary_key_bytes = &idx_key[TIME_KEY_LEN + 1..]; let primary_key = String::from_utf8(primary_key_bytes.to_vec()) @@ -317,7 +346,7 @@ impl LibraryDb { candidates.push((primary_key, ts)); } - self.resolve_candidates(candidates, limit) + self.resolve_candidates(&rtxn, candidates, limit) } /// Count the total number of entries in the database. @@ -326,16 +355,22 @@ impl LibraryDb { /// /// Returns an error if the database operation fails. pub fn count(&self) -> Result { - Ok(self.db.len()?) + let rtxn = self.env.read_txn()?; + let len = self.tracks.len(&rtxn).context("Failed to count entries")?; + Ok(usize::try_from(len).unwrap_or(usize::MAX)) } /// Resolve a list of (`primary_key`, timestamp) candidates into report entries. /// /// Sorts by timestamp descending, truncates to `limit`, and looks up each - /// primary key in the default tree. Returns results in reverse (oldest-first) - /// order so that most recently added entries appear at the bottom. + /// primary key in `tracks`. Returns results in reverse (oldest-first) order + /// so that most recently added entries appear at the bottom. + /// + /// Takes the caller's read transaction so a whole query is one consistent + /// snapshot rather than a transaction per lookup. fn resolve_candidates( &self, + rtxn: &RoTxn, mut candidates: Vec<(String, i64)>, limit: usize, ) -> Result> { @@ -345,7 +380,7 @@ impl LibraryDb { let mut results = Vec::with_capacity(candidates.len()); for (primary_key, _) in candidates { - if let Some(report) = self.get(&primary_key)? { + if let Some(report) = self.tracks.get(rtxn, &primary_key)? { results.push((primary_key, report)); } } @@ -359,17 +394,18 @@ impl LibraryDb { /// /// Returns an error if the database operation fails. pub fn find_by_artist(&self, prefix: &str, limit: usize) -> Result> { + let rtxn = self.env.read_txn()?; let prefix_lower = prefix.to_lowercase(); let mut candidates: Vec<(String, i64)> = Vec::new(); - for item in self.artist_tree.scan_prefix(prefix_lower.as_bytes()) { + for item in self.artist_db.prefix_iter(&rtxn, prefix_lower.as_bytes())? { let (idx_key, value) = item.context("Failed to read artist index")?; - let primary_key = extract_primary_key_string(&idx_key)?; - let ts = parse_timestamp(&value); + let primary_key = extract_primary_key_string(idx_key)?; + let ts = parse_timestamp(value); candidates.push((primary_key, ts)); } - self.resolve_candidates(candidates, limit) + self.resolve_candidates(&rtxn, candidates, limit) } /// Find tracks by key signature prefix (case-insensitive), up to `limit` items. @@ -378,17 +414,18 @@ impl LibraryDb { /// /// Returns an error if the database operation fails. pub fn find_by_key(&self, prefix: &str, limit: usize) -> Result> { + let rtxn = self.env.read_txn()?; let prefix_lower = prefix.to_lowercase(); let mut candidates: Vec<(String, i64)> = Vec::new(); - for item in self.key_tree.scan_prefix(prefix_lower.as_bytes()) { + for item in self.key_db.prefix_iter(&rtxn, prefix_lower.as_bytes())? { let (idx_key, value) = item.context("Failed to read key index")?; - let primary_key = extract_primary_key_string(&idx_key)?; - let ts = parse_timestamp(&value); + let primary_key = extract_primary_key_string(idx_key)?; + let ts = parse_timestamp(value); candidates.push((primary_key, ts)); } - self.resolve_candidates(candidates, limit) + self.resolve_candidates(&rtxn, candidates, limit) } /// Find tracks with BPM in the given range (inclusive), up to `limit` items. @@ -402,51 +439,53 @@ impl LibraryDb { max: f64, limit: usize, ) -> Result> { + let rtxn = self.env.read_txn()?; let mut candidates: Vec<(String, i64)> = Vec::new(); - // Sled sorts longer keys after their prefix, so a bare 8-byte bound would + // LMDB sorts longer keys after their prefix, so a bare 8-byte bound would // exclude keys like [max_be, 0x00, primary_key]. + let lower = min.to_be_bytes(); let mut upper = Vec::with_capacity(BPM_KEY_LEN + 1); upper.extend_from_slice(&max.to_be_bytes()); upper.push(0xFF); - for item in self - .bpm_tree - .range(min.to_be_bytes().as_slice()..upper.as_slice()) - { + // A `Range<&[u8]>` is not usable here: only the `Bound` pair form of + // `RangeBounds` accepts an unsized key type. + let range = ( + Bound::Included(lower.as_slice()), + Bound::Excluded(upper.as_slice()), + ); + for item in self.bpm_db.range(&rtxn, &range)? { let (idx_key, value) = item.context("Failed to read bpm index")?; let primary_key_bytes = &idx_key[BPM_KEY_LEN + 1..]; let primary_key = String::from_utf8(primary_key_bytes.to_vec()) .context("Invalid UTF-8 in primary key")?; - let ts = parse_timestamp(&value); + let ts = parse_timestamp(value); candidates.push((primary_key, ts)); } - self.resolve_candidates(candidates, limit) + self.resolve_candidates(&rtxn, candidates, limit) } /// Search tracks by regex pattern. /// - /// Checks the primary key first (cheap, no deserialization), then - /// falls back to checking individual report fields. Returns up to - /// `limit` matches (pass `0` for no limit). + /// Matches the primary key first, then the individual report fields. + /// Returns up to `limit` matches (pass `0` for no limit). /// /// # Errors /// /// Returns an error if the database operation fails. pub fn search(&self, pattern: &Regex, limit: usize) -> Result> { + let rtxn = self.env.read_txn()?; let lim = effective_limit(limit); let mut results = Vec::new(); - for item in &self.db { - let (key, value) = item.context("Failed to read entry")?; - let key_str = String::from_utf8(key.to_vec()).context("Invalid UTF-8 key")?; + for item in self.tracks.iter(&rtxn)? { + let (key, report) = item.context("Failed to read entry")?; - let report = deserialize_report(&value)?; - // Check the primary key and individual fields — avoids // allocating a display String for every non-matching entry. - let matches = pattern.is_match(&key_str) + let matches = pattern.is_match(key) || report .artist .as_deref() @@ -458,7 +497,7 @@ impl LibraryDb { .is_some_and(|b| pattern.is_match(&format!("{b:.1}"))); if matches { - results.push((key_str, report)); + results.push((key.to_string(), report)); if results.len() >= lim { break; } @@ -467,16 +506,6 @@ impl LibraryDb { Ok(results) } - - /// Flush pending writes to disk. - /// - /// # Errors - /// - /// Returns an error if the database operation fails. - pub fn flush(&self) -> Result<()> { - self.db.flush()?; - Ok(()) - } } #[cfg(test)] @@ -622,7 +651,7 @@ mod tests { assert_eq!(empty.to_string(), "Unknown - Unknown | BPM: - | Key: -"); } - // ---- Integration tests against a real sled database ---- + // ---- Integration tests against a real database ---- fn report(artist: &str, track: &str, bpm: f64, key: &str, ts: i64) -> (String, TrackReport) { ( @@ -648,10 +677,10 @@ mod tests { #[test] fn save_and_get_roundtrip() { let (_dir, db) = temp_db(); - let (key, rep) = report("Aphex Twin", "Xtal", 120.0, "A Minor", 1); - db.save(&key, &rep).unwrap(); + db.save_batch(&[report("Aphex Twin", "Xtal", 120.0, "A Minor", 1)]) + .unwrap(); - let got = db.get(&key).unwrap().unwrap(); + let got = db.get("Aphex Twin - Xtal").unwrap().unwrap(); assert_eq!(got.artist.as_deref(), Some("Aphex Twin")); assert_eq!(got.bpm, Some(120.0)); assert!(db.get("missing").unwrap().is_none()); @@ -661,12 +690,12 @@ mod tests { #[test] fn find_by_artist_is_prefix_and_case_insensitive() { let (_dir, db) = temp_db(); - let (k1, r1) = report("Aphex Twin", "Xtal", 120.0, "A Minor", 1); - let (k2, r2) = report("Aphex Twin", "Ageispolis", 110.0, "C Major", 2); - let (k3, r3) = report("Boards of Canada", "Roygbiv", 95.0, "C Major", 3); - db.save(&k1, &r1).unwrap(); - db.save(&k2, &r2).unwrap(); - db.save(&k3, &r3).unwrap(); + db.save_batch(&[ + report("Aphex Twin", "Xtal", 120.0, "A Minor", 1), + report("Aphex Twin", "Ageispolis", 110.0, "C Major", 2), + report("Boards of Canada", "Roygbiv", 95.0, "C Major", 3), + ]) + .unwrap(); let hits = db.find_by_artist("aphex", 0).unwrap(); assert_eq!(hits.len(), 2); @@ -682,12 +711,12 @@ mod tests { #[test] fn find_by_key_is_prefix_and_case_insensitive() { let (_dir, db) = temp_db(); - let (k1, r1) = report("A", "x", 120.0, "C Major", 1); - let (k2, r2) = report("B", "y", 110.0, "C Minor", 2); - let (k3, r3) = report("C", "z", 95.0, "D Major", 3); - db.save(&k1, &r1).unwrap(); - db.save(&k2, &r2).unwrap(); - db.save(&k3, &r3).unwrap(); + db.save_batch(&[ + report("A", "x", 120.0, "C Major", 1), + report("B", "y", 110.0, "C Minor", 2), + report("C", "z", 95.0, "D Major", 3), + ]) + .unwrap(); assert_eq!(db.find_by_key("c ", 0).unwrap().len(), 2); assert_eq!(db.find_by_key("c major", 0).unwrap().len(), 1); @@ -697,10 +726,12 @@ mod tests { #[test] fn find_by_bpm_range_is_inclusive() { let (_dir, db) = temp_db(); - for (i, bpm) in [90.0, 120.0, 140.0, 160.0].into_iter().enumerate() { - let (k, r) = report("artist", &format!("t{i}"), bpm, "C Major", i as i64); - db.save(&k, &r).unwrap(); - } + let entries: Vec<_> = [90.0, 120.0, 140.0, 160.0] + .into_iter() + .enumerate() + .map(|(i, bpm)| report("artist", &format!("t{i}"), bpm, "C Major", i as i64)) + .collect(); + db.save_batch(&entries).unwrap(); let hits = db.find_by_bpm_range(120.0, 140.0, 0).unwrap(); let mut bpms: Vec = hits.iter().map(|(_, r)| r.bpm.unwrap()).collect(); @@ -715,12 +746,12 @@ mod tests { #[test] fn list_orders_oldest_last_and_respects_limit() { let (_dir, db) = temp_db(); - let (k1, r1) = report("A", "old", 100.0, "C Major", 10); - let (k2, r2) = report("B", "mid", 100.0, "C Major", 20); - let (k3, r3) = report("C", "new", 100.0, "C Major", 30); - db.save(&k1, &r1).unwrap(); - db.save(&k2, &r2).unwrap(); - db.save(&k3, &r3).unwrap(); + db.save_batch(&[ + report("A", "old", 100.0, "C Major", 10), + report("B", "mid", 100.0, "C Major", 20), + report("C", "new", 100.0, "C Major", 30), + ]) + .unwrap(); // Full listing is chronological: oldest first, newest at the bottom. let all = db.list(0).unwrap(); @@ -736,29 +767,12 @@ mod tests { #[test] fn save_overwrites_and_refreshes_indexes() { let (_dir, db) = temp_db(); - let key = "Artist - Track".to_string(); - let first = TrackReport { - artist: Some("Artist".to_string()), - track: Some("Track".to_string()), - bpm: Some(120.0), - key: Some("C Major".to_string()), - timestamp: Some(1), - bpm_confidence: None, - key_strength: None, - }; - db.save(&key, &first).unwrap(); + db.save_batch(&[report("Artist", "Track", 120.0, "C Major", 1)]) + .unwrap(); // Overwrite with a different key signature and BPM. - let second = TrackReport { - artist: Some("Artist".to_string()), - track: Some("Track".to_string()), - bpm: Some(128.0), - key: Some("A Minor".to_string()), - timestamp: Some(2), - bpm_confidence: Some(4.1), - key_strength: Some(0.66), - }; - db.save(&key, &second).unwrap(); + db.save_batch(&[report("Artist", "Track", 128.0, "A Minor", 2)]) + .unwrap(); assert_eq!(db.count().unwrap(), 1); // Stale indexes are gone; only the new ones resolve. @@ -769,28 +783,10 @@ mod tests { } #[test] - fn delete_removes_entry_and_indexes() { - let (_dir, db) = temp_db(); - let (key, rep) = report("Aphex Twin", "Xtal", 120.0, "A Minor", 1); - db.save(&key, &rep).unwrap(); - db.delete(&key).unwrap(); - - assert!(db.get(&key).unwrap().is_none()); - assert_eq!(db.count().unwrap(), 0); - assert!(db.find_by_artist("aphex", 0).unwrap().is_empty()); - assert!(db.find_by_key("a minor", 0).unwrap().is_empty()); - assert!(db.find_by_bpm_range(119.0, 121.0, 0).unwrap().is_empty()); - assert!(db.list(0).unwrap().is_empty()); - - // Deleting a missing key is a no-op. - db.delete("does not exist").unwrap(); - } - - #[test] fn save_batch_inserts_all_and_refreshes_stale_indexes() { let (_dir, db) = temp_db(); - let (k, r) = report("Artist", "Track", 100.0, "C Major", 1); - db.save(&k, &r).unwrap(); + db.save_batch(&[report("Artist", "Track", 100.0, "C Major", 1)]) + .unwrap(); let batch = vec![ report("Artist", "Track", 128.0, "A Minor", 5), // overwrites k @@ -807,10 +803,11 @@ mod tests { #[test] fn search_matches_key_artist_track_and_bpm() { let (_dir, db) = temp_db(); - let (k1, r1) = report("Aphex Twin", "Xtal", 120.0, "A Minor", 1); - let (k2, r2) = report("Boards of Canada", "Roygbiv", 95.5, "C Major", 2); - db.save(&k1, &r1).unwrap(); - db.save(&k2, &r2).unwrap(); + db.save_batch(&[ + report("Aphex Twin", "Xtal", 120.0, "A Minor", 1), + report("Boards of Canada", "Roygbiv", 95.5, "C Major", 2), + ]) + .unwrap(); // Match on the primary key / artist. assert_eq!( @@ -835,4 +832,17 @@ mod tests { let both = db.search(&Regex::new(".").unwrap(), 1).unwrap(); assert_eq!(both.len(), 1); } + + #[test] + fn keys_may_exceed_the_stock_lmdb_limit() { + // LMDB caps keys at 511 bytes without heed's `longer-keys` feature, and + // an artist index key is the artist plus the whole primary key again. + let (_dir, db) = temp_db(); + let artist = "a".repeat(400); + let entry = report(&artist, "Long", 120.0, "C Major", 1); + db.save_batch(std::slice::from_ref(&entry)).unwrap(); + + assert!(db.get(&entry.0).unwrap().is_some()); + assert_eq!(db.find_by_artist(&artist, 0).unwrap().len(), 1); + } } blob - 85f4bd513178cad3183e1df4dda988e3968fa5ce blob + f52e5988a904d25e0ae5e2d870ca0017802ccaf7 --- src/file_io.rs +++ src/file_io.rs @@ -60,7 +60,6 @@ pub fn process_file(path: &Path, options: ProcessOptio path.display() ); return Ok(TrackInfo { - path: path.to_path_buf(), artist: metadata.artist, track: metadata.track, bpm, @@ -92,7 +91,6 @@ pub fn process_file(path: &Path, options: ProcessOptio } Ok(TrackInfo { - path: path.to_path_buf(), artist: metadata.artist, track: metadata.track, bpm, blob - e3ea471ba0d52eb20175c3a0812ea2536b470c22 blob + b7d5b0b7e0a5d665fc5aa5ab236caea50d5d7897 --- src/import.rs +++ src/import.rs @@ -21,15 +21,15 @@ fn convert_mixxx_key(raw: &str) -> Option { } } -/// Import tracks from a Mixxx `SQLite` database into the sled library database. +/// Import tracks from a Mixxx `SQLite` database into the library database. /// -/// Reads the `library` and `track_locations` tables, skipping entries marked as -/// deleted, and saves each track with its BPM and key. +/// Reads the `library` table, skipping entries marked as deleted, and saves +/// each track with its BPM and key. /// /// # Errors /// /// Returns an error if the Mixxx database cannot be opened or queried, or if -/// sled writes fail. +/// the import cannot be written. pub fn import_mixxx(db: &LibraryDb, mixxx_path: &str) -> Result { let conn = Connection::open(mixxx_path).with_context(|| "Failed to open Mixxx database")?; blob - e01f9c2160167e26dda96712c8b62b24dda1849b blob + 874be56ab0cadd2091794d0948f31fbe6576f195 --- src/main.rs +++ src/main.rs @@ -344,7 +344,6 @@ fn real_main() -> Result<()> { .collect(); db.save_batch(&entries)?; - db.flush()?; for report in entries.iter().map(|(_, r)| r) { println!(" Saved: {report}"); blob - 34fa20a7a10d18be906373da3d212be4c16cd51c blob + 4306d89879e3d5a0c0c86b922e8bb4d947e60837 --- src/types.rs +++ src/types.rs @@ -2,7 +2,6 @@ use std::path::PathBuf; #[derive(Debug)] pub struct TrackInfo { - pub path: PathBuf, pub artist: Option, pub track: Option, pub bpm: f32,