commit - 79b649367f789a4db56a62e6324a174d9c0b325b
commit + 0cc756c5727a93adc29fedbcab77cfefea7724fa
blob - 4d354047206c1350b95b254dfd6ba4c831cbad41
blob + a10869184935066d78d48b0212a866853c6b4b8f
--- flake.nix
+++ flake.nix
craneLib = (crane.mkLib pkgs).overrideToolchain stableToolchain;
# Essentia is not packaged as a library in nixpkgs (only the prebuilt
- # extractor binary), so we build it from source. See nix/essentia.nix.
- # essentia beta5 bundles waf 2.0.10, which imports the `imp` module
- # removed in Python 3.12, so build it with Python 3.11.
- essentia = pkgs.callPackage ./nix/essentia.nix {python3 = pkgs.python311;};
+ # extractor binary), so we build it from source. See nix/essentia.nix
+ # for why it tracks master rather than the v2.1_beta5 tag.
+ essentia = pkgs.callPackage ./nix/essentia.nix {};
+ # Every shared library virittaa ends up with a DT_NEEDED entry for.
+ # rustc emits `-L native=...` but no RPATH for any of them, so this list
+ # is needed twice: exported as LD_LIBRARY_PATH at build time (so
+ # essentia-codegen's build script, which links libessentia and is then
+ # *executed* to introspect it, can load), and again in the runtime
+ # wrapper.
+ runtimeLibs =
+ [
+ essentia
+ pkgs.ffmpeg
+ pkgs.fftwFloat
+ pkgs.taglib
+ pkgs.libsamplerate
+ pkgs.chromaprint
+ pkgs.libyaml
+ pkgs.zlib
+ # rusqlite/libsqlite3-sys builds unbundled here, so it links the
+ # system libsqlite3 rather than compiling its own.
+ pkgs.sqlite
+ ]
+ # The cxx bridge drags in libstdc++.so.6.
+ ++ pkgs.lib.optionals (!pkgs.stdenv.isDarwin) [pkgs.stdenv.cc.cc.lib];
+
# virittaa's Cargo.toml references the essentia-rs crate as a sibling path
# dependency (`../essentia-rs/essentia`). That sibling does not exist in
# the build sandbox, so assemble a source tree that contains both crates
chmod -R u+w $out/nix-vendor-essentia-rs
substituteInPlace $out/Cargo.toml \
--replace-fail '../essentia-rs/essentia' 'nix-vendor-essentia-rs/essentia'
+
+ # essentia_sys's build.rs unconditionally appends "eigen3" to every
+ # include path eigen3.pc reports. That is right on Debian, whose
+ # eigen3.pc says `-I/usr/include`, but nixpkgs' already says
+ # `-I$out/include/eigen3` -- so the bridge gets `include/eigen3/eigen3`
+ # and cannot find <unsupported/Eigen/CXX11/Tensor>. Only append when
+ # the reported path is not already the Eigen root.
+ #
+ # This belongs upstream in essentia-rs; when it lands there,
+ # --replace-fail will start failing and this block can go.
+ substituteInPlace $out/nix-vendor-essentia-rs/essentia_sys/build.rs \
+ --replace-fail 'if library.name == "eigen3" {' \
+ 'if library.name == "eigen3" && !include_path.join("Eigen").is_dir() {'
'';
baseNativeBuildInputs = with pkgs; [
pkg-config
- cmake
llvmPackages.libclang
];
"-isystem ${pkgs.llvmPackages.libclang.lib}/lib/clang/${pkgs.llvmPackages.libclang.version}/include"
+ pkgs.lib.optionalString (!pkgs.stdenv.isDarwin) " -isystem ${pkgs.glibc.dev}/include";
- nativeBuildInputs =
- baseNativeBuildInputs
- ++ pkgs.lib.optionals (!pkgs.stdenv.isDarwin) [pkgs.glibc.dev];
+ nativeBuildInputs = baseNativeBuildInputs;
+ # essentia-codegen's build script is linked against libessentia and
+ # then run by cargo to introspect the algorithm registry. rustc emits
+ # `-L native=...` but no RPATH, so the loader cannot find the .so:
+ # "error while loading shared libraries: libessentia.so". The upstream
+ # fix is for essentia_sys's build.rs to emit
+ # `cargo:rustc-link-arg=-Wl,-rpath,<libdir>` for each probed library,
+ # which would also make the final binary self-contained and retire the
+ # wrapper below.
+ preBuild = ''
+ export LD_LIBRARY_PATH="${pkgs.lib.makeLibraryPath runtimeLibs}''${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
+ '';
+
buildInputs =
[
# essentia-rs's essentia_sys probes each of these via pkg-config at
# builds against, so the store paths match libessentia's.
essentia
pkgs.eigen
- pkgs.ffmpeg_4
+ pkgs.ffmpeg
pkgs.fftwFloat
- pkgs.taglib_1
+ pkgs.taglib
pkgs.libsamplerate
pkgs.chromaprint
pkgs.libyaml
# taglib's pkg-config Libs pulls in -lz.
pkgs.zlib
+ # libsqlite3-sys links the system sqlite (no `bundled` feature).
+ pkgs.sqlite
]
++ pkgs.lib.optionals pkgs.stdenv.isDarwin [pkgs.libiconv];
};
cargoArtifacts = null;
nativeBuildInputs = commonArgs.nativeBuildInputs ++ [pkgs.makeWrapper];
postInstall = ''
- wrapProgram $out/bin/virittaa --prefix LD_LIBRARY_PATH : "${pkgs.lib.makeLibraryPath [
- essentia
- pkgs.ffmpeg_4
- pkgs.fftwFloat
- pkgs.taglib_1
- pkgs.libsamplerate
- pkgs.chromaprint
- pkgs.libyaml
- ]}"
+ # The build-time failure above shows rustc does not add an RPATH
+ # for these, so the installed binary needs the same help. Verify
+ # with `patchelf --print-rpath result/bin/.virittaa-wrapped`;
+ # if the rpath ever covers them, drop this and makeWrapper --
+ # LD_LIBRARY_PATH leaks into anything virittaa execs.
+ wrapProgram $out/bin/virittaa \
+ --prefix LD_LIBRARY_PATH : "${pkgs.lib.makeLibraryPath runtimeLibs}"
'';
});
in {
inherit virittaa essentia;
};
+ # `nix flake check` only. Note that clippy gets no dependency cache
+ # (cargoArtifacts = null, see above), so it is a second full build of
+ # the tree -- expensive, but opt-in. A cargoTest check is deliberately
+ # absent: cleanCargoSource strips non-Rust files, so any test fixture
+ # would go missing.
checks = {
inherit virittaa;
+ fmt = craneLib.cargoFmt {inherit src;};
+ clippy = craneLib.cargoClippy (commonArgs
+ // {
+ cargoArtifacts = null;
+ cargoClippyExtraArgs = "--all-targets -- --deny warnings";
+ });
};
devShells.default = pkgs.mkShell {
blob - 9d9cfa19999f876c078acbdcb97e1b2d1f0c9d8b
blob + 1bcac2c2dfeb8d9ac17ab25daee37af9d6a4b6fd
--- nix/BUILD_NOTES.md
+++ nix/BUILD_NOTES.md
them: `essentia eigen3 yaml-0.1 fftw3f taglib samplerate libchromaprint
libavformat libswresample libavcodec libavutil` — plus **tensorflow unless
`USE_TENSORFLOW=0`**. Note it links **swresample** (modern ffmpeg), not
- avresample.
+ avresample; that is why essentia itself must be built from master, not from
+ the v2.1_beta5 tag (see below).
## Flake structure (flake.nix)
Cargo.toml path `../essentia-rs/essentia` → `nix-vendor-essentia-rs/essentia`.
essentia-rs is copied **raw** (not cleanCargoSource) because its C++ bridge
files (`essentia_sys/bridge/*.cpp,*.h`) would be stripped by the cargo filter.
+- The same step patches `essentia_sys/build.rs`: it appends `eigen3` to every
+ include path `eigen3.pc` reports, which is correct on Debian (`-I/usr/include`)
+ but wrong on nixpkgs (`-I$out/include/eigen3` already), yielding
+ `include/eigen3/eigen3` and `fatal error: unsupported/Eigen/CXX11/Tensor: No
+ such file or directory`. **This is an essentia-rs bug** — fix it there
+ (`if library.name == "eigen3" && !include_path.join("Eigen").is_dir()`), then
+ drop the patch here; `--replace-fail` will flag it once upstream changes.
- `USE_TENSORFLOW = "0"` set in `commonArgs` (also in `.cargo/config.toml` for
non-Nix builds).
- Single-shot build: `cargoArtifacts = null` (no crane buildDepsOnly). The dep
split is skipped because crane's dummy-source build would strip essentia-rs's
- C++ files and can't run the codegen.
+ C++ files and can't run the codegen. Cost: every rebuild recompiles the whole
+ dep tree.
- `buildInputs` lists the full pkg-config set explicitly (essentia, eigen,
- ffmpeg_4, fftwFloat, taglib_1, libsamplerate, chromaprint, libyaml) so the
+ ffmpeg, fftwFloat, taglib, libsamplerate, chromaprint, libyaml, zlib) so the
essentia_sys probes always resolve, rather than relying on essentia's
propagatedBuildInputs (multi-output `.dev` propagation is unreliable).
- `essentia` is exposed as a package output: `nix build .#essentia`.
+- No `cmake` in nativeBuildInputs — nothing in the lockfile needs it (zstd-sys
+ and libsqlite3-sys both use `cc`). No `glibc.dev` either; the only thing that
+ wanted it was bindgen, which gets it via `BINDGEN_EXTRA_CLANG_ARGS`.
+- `checks` adds `fmt` and `clippy`. `clippy` is a second uncached full build,
+ so it only runs under `nix flake check`, never `nix build`.
## nix/essentia.nix
-- essentia `v2.1_beta5`, driven by its bundled `waf`.
-- **Must build with Python 3.11**: waf 2.0.10 does `import imp`, removed in
- Python 3.12 (nixpkgs-unstable default). flake.nix calls it with
- `{ python3 = pkgs.python311; }`. Symptom if this regresses:
- `ModuleNotFoundError: No module named 'imp'` during configure.
-- propagatedBuildInputs: fftwFloat ffmpeg_4 libsamplerate taglib_1 chromaprint
- libyaml. (ffmpeg_4 also provides libswresample, so it's consistent with what
- essentia_sys links.)
-- installPhase patches essentia.pc Cflags to add `-I${includedir}`.
+- essentia is pinned to a **master commit** (`b9fa6cb`, 2.1-beta6-dev,
+ 2026-05-20), not the `v2.1_beta5` tag. beta5 is a dead end: it needs
+ `libavresample` (removed in ffmpeg 5, so it forces `ffmpeg_4`) and bundles
+ waf 2.0.10, which needs Python ≤ 3.11 (`import imp`) *and* source patching
+ for the `'rU'` open mode Python 3.11 removed. Master uses `libswresample`
+ and bundles waf 2.0.25, which runs unmodified on Python 3.14.
+- Consequently: plain `ffmpeg`, plain `taglib` (2.x satisfies the wscript's
+ `taglib >= 1.9` check, and the API essentia uses — `FileRef`, `PropertyMap`,
+ `StringList` — survived the taglib 2.0 break), plain `python3`. No pins.
+- `zlib` is required: taglib's pkg-config `Libs` pulls in `-lz` and nothing
+ else in the closure provides it. Symptom if it regresses: all 288 objects
+ compile, then `ld.bfd: cannot find -lz` while linking `libessentia.so`.
+- propagatedBuildInputs: eigen fftwFloat ffmpeg libsamplerate taglib
+ chromaprint libyaml zlib.
+- `eigen` is the one *mandatory* pkg-config check on master (`eigen3 >= 3.3.4`);
+ every other `check_cfg` in `src/wscript` is `mandatory=False`, so a missing
+ dep there silently drops algorithms instead of failing the build. Worth
+ reading the "CONFIGURATION SUMMARY" block in the log — MonoLoader,
+ MetadataReader, Resample etc. disappear if ffmpeg/taglib/samplerate go
+ missing, and you only find out when the Rust codegen can't find them.
+- installPhase patches essentia.pc Cflags to add `-I${includedir}`
+ (essentia.pc.in still lists only the per-subdirectory include paths).
+- `postPatch` makes `scheduler::Network::lastCreated` `thread_local`. Upstream
+ it is one process-global pointer, written by every Network ctor/dtor and read
+ by every `Network::run()`. MonoLoader, RhythmExtractor2013 and KeyExtractor
+ each build an inner Network, so analysing with `-j >1` corrupts it: warning
+ spam ("No network created..."), interleaved output, then `free(): double free
+ detected in tcache`. The same hunk adds the missing `return` after that
+ warning, which otherwise falls straight into a null dereference. Essentia
+ supports one-instance-per-thread otherwise (FFTW plans and the devnull/copy
+ singletons are ForcedMutex-guarded); this global is the outlier. **Report
+ upstream.** `src/audio_processing.rs` documents that it depends on this.
+- To move the pin: `nix hash path --sri --type sha256 <unpacked tree>`, or set
+ `hash = lib.fakeHash` and read the correct value out of the build error.
## Current status
- [x] Rust integration done (`src/audio_processing.rs` uses
MonoLoader → RhythmExtractor2013(multifeature) → KeyExtractor(EDMA)).
- [x] flake restructured (path-dep vendoring, single-shot, USE_TENSORFLOW=0).
-- [~] `nix build .#essentia` — FIRST attempt failed on the waf/`imp` Python 3.12
- issue. Fix applied (python311). **Rebuild in progress / next step.**
-- [ ] `nix build .#virittaa` — not yet attempted. Watch for: essentia-codegen
- running libessentia at build time (needs rpath/LD path to essentia + its
- deps); the generated module paths / enum variants assumed in
+- [x] `nix build .#essentia` — builds clean on master against ffmpeg 8.1,
+ taglib 2.2.1, eigen 3.4.1, Python 3.14.
+- [x] The C++ bridge compiles (after the eigen include fix above).
+- [x] essentia-codegen's build script needs `LD_LIBRARY_PATH` — rustc emits
+ `-L native=...` but no RPATH, so *running* the build script fails with
+ "libessentia.so: cannot open shared object file", and once that resolves,
+ "libstdc++.so.6: ..." (the cxx bridge). Both come from the shared
+ `essentiaLibs` list, exported in `preBuild` and reused by the runtime
+ wrapper; `stdenv.cc.cc.lib` is in it for libstdc++.
+ Upstream fix: have essentia_sys's build.rs emit
+ `cargo:rustc-link-arg=-Wl,-rpath,<libdir>` per probed library; that would
+ retire both the preBuild export and the wrapper.
+- [x] `nix build .#virittaa` — builds and links. The codegen ran against
+ master's libessentia, so the module paths and enum variants in
audio_processing.rs (`essentia::algorithm::{input_output,rhythm,tonal}`,
`RhythmExtractor2013Method::Multifeature`, `KeyExtractorProfileType::Edma`)
- — these are unverified until this first real compile.
+ are confirmed correct. `sqlite` had to be added to buildInputs:
+ libsqlite3-sys builds unbundled and links the system library.
+- [ ] Verify `-j >1` analysis is now clean end to end (this is what surfaced the
+ `lastCreated` bug above).
+- [ ] Drop the `LD_LIBRARY_PATH` wrapper in `postInstall` once the built binary
+ is confirmed to carry the right RPATH (`patchelf --print-rpath
+ result/bin/.virittaa-wrapped`).
## Reproduce / continue
```sh
cd /home/miro/src/sr.ht/virittaa
git add -A # flake (git) only sees tracked/staged files!
-nix build .#essentia -L # should now pass with python311
+nix build .#essentia -L # libessentia alone
nix build .#virittaa -L # the real test
```
blob - 2dd586e859cb9f4ad4ff93df3ff1784d5080ceff
blob + b7cc49ef2fa3e553dd0c918446c23983bebb430b
--- nix/essentia.nix
+++ nix/essentia.nix
-# Pinned to v2.1_beta5, which requires libavresample (removed in ffmpeg >= 5),
-# so we build against ffmpeg_4. taglib is pinned to the 1.x series for the same
-# API-compatibility reason.
+# Pinned to a master commit (2.1-beta6-dev) rather than the last tag, v2.1_beta5.
+# beta5 requires libavresample (removed in ffmpeg 5) and bundles waf 2.0.10,
+# which needs Python <= 3.11 (`import imp`) and patching for the removed 'rU'
+# open mode. Master resamples with libswresample instead -- matching what
+# essentia-rs's essentia_sys probes via pkg-config -- and bundles waf 2.0.25,
+# which runs unmodified on current Python. Both pins (ffmpeg_4, taglib_1) and
+# the whole waf patching dance are gone as a result.
{
lib,
stdenv,
fetchFromGitHub,
python3,
pkg-config,
+ eigen,
fftwFloat,
- ffmpeg_4,
+ ffmpeg,
libsamplerate,
- taglib_1,
+ taglib,
chromaprint,
libyaml,
zlib,
}:
stdenv.mkDerivation (finalAttrs: {
pname = "essentia";
- version = "2.1_beta5";
+ version = "2.1-beta6-dev-unstable-2026-05-20";
src = fetchFromGitHub {
owner = "MTG";
repo = "essentia";
- rev = "v${finalAttrs.version}";
- sha256 = "0ig267wfxry2rjpij4fypf2zk7f59jd0n23l3807igbn2cmkgz4w";
+ rev = "b9fa6cb674ca43dfb94d28d293aeda441c6745db";
+ hash = "sha256-8z7SZN6tO/3iB2bOzPnz1en0NjbgGJcTjkj02PzvdVs=";
};
+ # `scheduler::Network::lastCreated` is a process-global pointer, assigned by
+ # every Network constructor and cleared by every destructor, and read on every
+ # `Network::run()` via printNetworkBufferFillState(). Since MonoLoader,
+ # RhythmExtractor2013 and KeyExtractor each build an inner Network, analysing
+ # files on more than one thread makes threads clobber each other's pointer:
+ # "No network created, or last created network has been deleted..." spam,
+ # interleaved output, then a use-after-free ("double free detected in tcache").
+ #
+ # It is purely a debug convenience, so make it per-thread. Essentia otherwise
+ # supports one-instance-per-thread use (it guards FFTW plan creation and the
+ # devnull/copy singletons with ForcedMutex); this global is the outlier.
+ # The second hunk fixes the null dereference on the line right after the
+ # warning that says the pointer is null. Both are worth reporting upstream.
+ postPatch = ''
+ substituteInPlace src/essentia/scheduler/network.h \
+ --replace-fail 'static Network* lastCreated;' \
+ 'static thread_local Network* lastCreated;'
+ substituteInPlace src/essentia/scheduler/network.cpp \
+ --replace-fail 'Network* Network::lastCreated = 0;' \
+ 'thread_local Network* Network::lastCreated = 0;' \
+ --replace-fail 'E_WARNING("No network created, or last created network has been deleted...");' \
+ 'E_WARNING("No network created, or last created network has been deleted..."); return;'
+ '';
+
+ strictDeps = true;
+
nativeBuildInputs = [
- python3
+ python3 # only runs waf; no Python bindings are built
pkg-config
];
# Propagated so anything linking libessentia.so also picks up its runtime
# dependencies (the .so records them as NEEDED).
propagatedBuildInputs = [
+ # The only hard dependency on master (`eigen3 >= 3.3.4`, mandatory); every
+ # other check_cfg in src/wscript is mandatory=False. Header-only, but
+ # propagated because essentia's own headers include it.
+ eigen
fftwFloat
- ffmpeg_4
+ ffmpeg
libsamplerate
- taglib_1
+ taglib
chromaprint
libyaml
# taglib's pkg-config Libs pulls in -lz; nothing else in the closure
zlib
];
- # waf 2.0.10's bundled waflib opens files with the universal-newline 'rU'
- # mode, which Python 3.11 removed, so waf fails with "invalid mode: 'rUb'".
- # The bundled waf also imports the removed 'imp' module, so it must be driven
- # by Python 3.11 (passed in as pkgs.python311). We let waf unpack itself
- # once, then drop the 'U' flag from every call site (Context.py reads
- # wscripts during configure, ConfigSet.py reads the lockfile at the start of
- # every later invocation).
configurePhase = ''
runHook preConfigure
+ # waf caches its unpacked waflib under $HOME.
export HOME=$TMPDIR
-
- # Unpack the bundled waflib. Python 3.11 can still decode the archive,
- # but it cannot use 'rU' once configure starts reading wscript files.
- python3 waf --version > /dev/null
-
- for f in .waf3-*/waflib/Context.py .waf3-*/waflib/ConfigSet.py; do
- substituteInPlace "$f" --replace-fail "'rU'" "'r'"
- done
-
- # Use the patched waflib for this and the later build/install phases.
- export WAFDIR="$PWD/$(echo .waf3-*)"
-
python3 waf configure --prefix=$out
runHook postConfigure
'';
installPhase = ''
runHook preInstall
python3 waf install
+ # essentia.pc.in lists only the per-subdirectory include paths, so
+ # `#include <essentia/...>` (what the essentia-rs bridge writes) does not
+ # resolve. Add the parent include dir.
substituteInPlace $out/lib/pkgconfig/essentia.pc \
--replace-fail 'Cflags: -I''${includedir}/essentia' \
'Cflags: -I''${includedir} -I''${includedir}/essentia'
blob - 6fc9c4084df4226d1ef7990dfb0dd8d4b10907df
blob + 97df38831b5402d041b70c32b12fc7c768090ced
--- src/audio_processing.rs
+++ src/audio_processing.rs
/// composite extractors below also instantiate inner algorithms through it while
/// configuring. virittaa analyzes files in parallel (rayon), so we serialize the
/// create+configure phase; the heavy `compute` calls run unlocked.
+///
+/// Running `compute` unlocked additionally requires an Essentia whose
+/// `scheduler::Network::lastCreated` is `thread_local`. Upstream it is a single
+/// process-global pointer that every inner Network's constructor and destructor
+/// writes and every `Network::run()` reads, so concurrent analysis corrupts it
+/// and aborts with a double free. `nix/essentia.nix` patches it; when building
+/// against a distro Essentia, either apply the same patch or hold this lock
+/// across the whole function.
static FACTORY_LOCK: Mutex<()> = Mutex::new(());
/// Result of analysing a single track.
