commit 6d9fd2e2cb0b7353b9d266ecd6d93ff53f6cb2fd from: mtmn date: Mon Jul 27 18:49:43 2026 UTC consolidate build scripts, cleanup commit - e6001f45f7b64607639d592e3e13c555e46a1a89 commit + 6d9fd2e2cb0b7353b9d266ecd6d93ff53f6cb2fd blob - 65b4df5b4b9c3ca4ad9b45bf7d7fb813e78d5a06 blob + 4075fb24e83adb9df763f0eaa25b60c73b1c0a37 --- .gitignore +++ .gitignore @@ -1,6 +1,2 @@ target result -zilinaplayed -essentia -void-runbook.org -nix/BUILD_NOTES.md blob - 012799ab2ff622e7381b2447d514e598ec3582dc blob + 5a8915ba70fd91906de9a173677f2a330ebda15d --- Cargo.toml +++ Cargo.toml @@ -15,7 +15,7 @@ sled = { git = "https://github.com/spacejam/sled", rev dirs = "6" regex = "1" rusqlite = "0.40" -essentia = { path = "../essentia-rs/essentia" } +essentia = { git = "ssh://anonymous@mtmn.name/essentia-rs.git" } [dev-dependencies] tempfile = "3" blob - 5a5754dbeec65139a77ea3da7d958a352f80f72f blob + 12e7345d55f8e7c284aaa325fb98522a031cc2e2 --- README.md +++ README.md @@ -1,116 +1,95 @@ # virittaa [![builds.sr.ht status](https://builds.sr.ht/~mtmn/virittaa.svg)](https://builds.sr.ht/~mtmn/virittaa?) -A command-line tool that analyzes audio files to detect `BPM` and `Key` tags. +Command-line BPM and key detection for audio files, backed by +[Essentia](https://github.com/MTG/essentia): tempo from `RhythmExtractor2013` +(multifeature), key from `KeyExtractor` (EDMA profile). The reported BPM is +Essentia's tempo estimate, not folded into a fixed octave range. -BPM and key detection are powered by [Essentia](https://github.com/MTG/essentia): -tempo comes from `RhythmExtractor2013` (multifeature method) and key from -`KeyExtractor` using the EDMA profile. The reported BPM is Essentia's true tempo -estimate — there is no octave-folding into a fixed range. +Detected values can be written back as tags and are stored in a local database +that can be queried by artist, key, BPM range, or regex. Mixxx libraries can be +imported from their SQLite database. -## Features - -* Detect BPM and Key of audio files -* Write detected BPM and Key to metadata tags -* Save track metadata to a database (on by default) -* Query the database by artist, key, BPM range, or regex -* Import tracks from a Mixxx SQLite database - ## Building -virittaa binds to Essentia through the [`essentia`](https://github.com/lagmoellertim/essentia-rs) -crate (essentia-rs), which links against a system-installed `libessentia` and -its dependencies via `pkg-config`. Before building, make sure all of the -following resolve (`pkg-config --exists `): +virittaa binds Essentia through the +[`essentia`](https://github.com/lagmoellertim/essentia-rs) crate (essentia-rs), +which links a system `libessentia` and its dependencies via `pkg-config`. These +must resolve: `essentia`, `eigen3`, `yaml-0.1`, `fftw3f`, `taglib`, `samplerate`, `libchromaprint`, `libavformat`, `libswresample`, `libavcodec`, `libavutil` -Essentia is not usually packaged as a library, so you will likely need to build -and install it from source (see `nix/essentia.nix` for a reference build). A -C++17 compiler is also required (the bindings are generated and compiled with -`cxx`). +Essentia is rarely packaged as a library, so it generally has to be built from +source. Build it from a recent `master`, not the `v2.1_beta5` tag: beta5 needs +`libavresample`, removed in ffmpeg 5, while essentia-rs probes for +`libswresample`. Apply `nix/essentia-network-thread-local.patch` — without it, +analysis on more than one thread aborts with a double free. -essentia-rs links TensorFlow by default; virittaa doesn't need it and opts out -via `USE_TENSORFLOW=0`, which is set for you in `.cargo/config.toml`. +A C++17 compiler is required; the bindings are generated and compiled with +`cxx`. essentia-rs links TensorFlow unless `USE_TENSORFLOW=0`, which +`.cargo/config.toml` sets. -To build the project, run: +essentia-rs is a git dependency (`ssh://anonymous@mtmn.name/essentia-rs.git`), +so the remote must be reachable over ssh: ```sh cargo build --release ``` +### Nix + +```sh +nix build .#virittaa # or .#essentia for the library alone +nix flake check # fmt + clippy, a second full build of the tree +``` + +The flake builds libessentia from `nix/essentia.nix`, vendors essentia-rs from +its `essentia-rs` input, and repoints the manifest at the vendored copy, so the +sandbox needs no network. virittaa's own sources come from git — untracked files +are invisible to the build. + +### Void Linux + +`hack/void-build.rb`, as root, builds libessentia and virittaa in a Void chroot +and emits a relocatable bundle: the binary plus its private libraries under an +`$ORIGIN` rpath, runnable from anywhere without `LD_LIBRARY_PATH`, and installed +into the invoking user's `~/.local` unless `-n`. Options: `--help`. + ## Usage ```sh -./target/release/virittaa [OPTIONS] [PATH]... +virittaa [OPTIONS] [PATH]... ``` -### Arguments +`[PATH]...` — files or directories to analyze; omit when querying the database. -* `[PATH]...`: Files or directories to analyze. Not required when using database query flags. +| Option | | +| --- | --- | +| `-t`, `--write-tags` | Write detected BPM and key to file metadata | +| `--no-store` | Do not save reports to the database | +| `-f`, `--force` | Re-analyze files that already carry BPM/key tags | +| `-j`, `--jobs ` | Thread count (0 = all cores) | +| `-l`, `--list` | List reports in the database | +| `--limit ` | Cap listed entries (default 1000, 0 = all) | +| `--query ` | Look up one track by exact key | +| `-S`, `--search ` | Search reports by regex | +| `-a`, `--artist ` | Match on artist prefix | +| `--key ` | Match on key prefix, e.g. `C Major`, `Am` | +| `--bpm ` | Match on BPM range | +| `--import-mixxx ` | Import from a Mixxx SQLite database | +| `--db-path ` | Database directory (default `~/.local/share/virittaa`) | -### Options +Analysis saves to the database unless `--no-store`, and skips files that already +have both tags unless `--force`. -* `-t`, `--write-tags`: Write detected BPM and Key to audio file metadata. -* `--no-store`: Skip saving track reports to the database. -* `-f`, `--force`: Re-analyze files that already have BPM/Key tags. -* `-j`, `--jobs `: Number of threads (0 = all cores). -* `-l`, `--list`: List track reports in the database. -* `--limit `: Max entries to list (default: 1000, 0 = all). -* `--query `: Look up a track by exact key. -* `-S`, `--search `: Search track reports by regex. -* `-a`, `--artist `: Find tracks by artist name prefix. -* `--key `: Find tracks by key prefix (e.g. 'C Major', 'Am'). -* `--bpm `: Find tracks with BPM in range MIN MAX. -* `--import-mixxx `: Import tracks from a Mixxx SQLite database. -* `--db-path `: Database directory (default: `~/.local/share/virittaa`). -* `-h`, `--help`: Print help information. -* `-V`, `--version`: Print version information. - -### Examples - ```sh -# Analyze a single file (saves to DB by default) -./target/release/virittaa /path/to/track.mp3 - -# Analyze a directory and write metadata tags -./target/release/virittaa --write-tags /path/to/music/ - -# Re-analyze files that already have tags -./target/release/virittaa --force --write-tags /path/to/music/ - -# Analyze without saving to the database -./target/release/virittaa --no-store /path/to/music/ - -# Limit to 4 threads -./target/release/virittaa -j 4 /path/to/music/ - -# Write tags and use a custom database path -./target/release/virittaa -t --db-path ~/my_library /path/to/music/ - -# List tracks in the database (default limit: 1000) -./target/release/virittaa --list - -# List all tracks (no limit) -./target/release/virittaa --list --limit 0 - -# Look up a specific track -./target/release/virittaa --query "Aphex Twin - Xtal" - -# Search by regex -./target/release/virittaa --search 'weed' -./target/release/virittaa --search '^Aphex' - -# Find all tracks by an artist -./target/release/virittaa --artist 'Aphex Twin' - -# Find all tracks in a key -./target/release/virittaa --key 'C Major' - -# Find tracks with BPM between 120 and 140 -./target/release/virittaa --bpm 120 140 - -# Import from a Mixxx database -./target/release/virittaa --import-mixxx ~/.mixxx/mixxxdb.sqlite +virittaa -tf ~/music/ # analyze, overwrite existing tags +virittaa -j 4 --no-store track.mp3 # 4 threads, nothing stored +virittaa --list --limit 0 # whole database +virittaa --query 'Aphex Twin - Xtal' +virittaa --search '^Aphex' +virittaa --key 'C Major' +virittaa --bpm 120 140 +virittaa --import-mixxx ~/.mixxx/mixxxdb.sqlite ``` blob - be6cc62a179c5c466647c76d5020dc8adc9dd7a8 blob + e1ef264c02d9944b86913351622319314bb4b04c --- flake.lock +++ flake.lock @@ -24,11 +24,11 @@ "rev": "9470b195fe6e4d522289221c02e066115bdf745d", "revCount": 58, "type": "git", - "url": "file:///home/miro/src/sr.ht/essentia-rs" + "url": "ssh://anonymous@mtmn.name/essentia-rs.git" }, "original": { "type": "git", - "url": "file:///home/miro/src/sr.ht/essentia-rs" + "url": "ssh://anonymous@mtmn.name/essentia-rs.git" } }, "fenix": { blob - a10869184935066d78d48b0212a866853c6b4b8f blob + f1a796e63a2ee838357c842305b210ca22c27728 --- flake.nix +++ flake.nix @@ -8,16 +8,11 @@ inputs.nixpkgs.follows = "nixpkgs"; }; - crane = { - url = "github:ipetkov/crane"; - }; + crane.url = "github:ipetkov/crane"; - # virittaa binds to Essentia through the essentia-rs crate, which lives in a - # sibling repo. It is not a flake, so we consume its source directly. Point - # this at the sr.ht remote (git+https://git.sr.ht/~mtmn/essentia-rs) once it - # is published. + # Vendored into the build tree by `src` below; cargo cannot fetch it there. essentia-rs = { - url = "git+file:///home/miro/src/sr.ht/essentia-rs"; + url = "git+ssh://anonymous@mtmn.name/essentia-rs.git"; flake = false; }; }; @@ -34,24 +29,10 @@ system: let pkgs = nixpkgs.legacyPackages.${system}; fenixPkgs = fenix.packages.${system}; - # Use the default rustup profile instead of the full toolchain so we do not - # pull in rust-docs and other components that are not required to build. - # This removes rust-docs from the build closure; it was the first derivation failing - # on the remote builder because the builder cannot substitute missing deps. stableToolchain = fenixPkgs.stable.defaultToolchain; craneLib = (crane.mkLib pkgs).overrideToolchain stableToolchain; - # Essentia is not packaged as a library in nixpkgs (only the prebuilt - # extractor binary), so we build it from source. See nix/essentia.nix - # for why it tracks master rather than the v2.1_beta5 tag. essentia = pkgs.callPackage ./nix/essentia.nix {}; - - # Every shared library virittaa ends up with a DT_NEEDED entry for. - # rustc emits `-L native=...` but no RPATH for any of them, so this list - # is needed twice: exported as LD_LIBRARY_PATH at build time (so - # essentia-codegen's build script, which links libessentia and is then - # *executed* to introspect it, can load), and again in the runtime - # wrapper. runtimeLibs = [ essentia @@ -62,47 +43,33 @@ pkgs.chromaprint pkgs.libyaml pkgs.zlib - # rusqlite/libsqlite3-sys builds unbundled here, so it links the - # system libsqlite3 rather than compiling its own. pkgs.sqlite ] - # The cxx bridge drags in libstdc++.so.6. ++ pkgs.lib.optionals (!pkgs.stdenv.isDarwin) [pkgs.stdenv.cc.cc.lib]; - # virittaa's Cargo.toml references the essentia-rs crate as a sibling path - # dependency (`../essentia-rs/essentia`). That sibling does not exist in - # the build sandbox, so assemble a source tree that contains both crates - # and rewrite the dependency path to point at the vendored copy. The raw - # essentia-rs source is used (not cleanCargoSource) because its - # essentia_sys crate ships C++ bridge files (*.cpp/*.h) that the cargo - # source filter would otherwise strip. + essentiaRsUrl = "ssh://anonymous@mtmn.name/essentia-rs.git"; + src = pkgs.runCommandLocal "virittaa-src" {} '' cp -r ${craneLib.cleanCargoSource ./.} $out chmod -R u+w $out cp -r ${essentia-rs} $out/nix-vendor-essentia-rs chmod -R u+w $out/nix-vendor-essentia-rs substituteInPlace $out/Cargo.toml \ - --replace-fail '../essentia-rs/essentia' 'nix-vendor-essentia-rs/essentia' + --replace-fail 'essentia = { git = "${essentiaRsUrl}" }' \ + 'essentia = { path = "nix-vendor-essentia-rs/essentia" }' - # essentia_sys's build.rs unconditionally appends "eigen3" to every - # include path eigen3.pc reports. That is right on Debian, whose - # eigen3.pc says `-I/usr/include`, but nixpkgs' already says - # `-I$out/include/eigen3` -- so the bridge gets `include/eigen3/eigen3` - # and cannot find . Only append when - # the reported path is not already the Eigen root. - # - # This belongs upstream in essentia-rs; when it lands there, - # --replace-fail will start failing and this block can go. + # Yields a path-resolved lock, matching the rewrite above, so --locked + # holds and crane skips the refetch. + sed -i '\|^source = "git+${essentiaRsUrl}|d' $out/Cargo.lock + + # essentia_sys appends "eigen3" to every include path eigen3.pc + # reports: right on Debian (`-I/usr/include`), wrong on nixpkgs + # (already `-I$out/include/eigen3`). Belongs upstream. substituteInPlace $out/nix-vendor-essentia-rs/essentia_sys/build.rs \ --replace-fail 'if library.name == "eigen3" {' \ 'if library.name == "eigen3" && !include_path.join("Eigen").is_dir() {' ''; - baseNativeBuildInputs = with pkgs; [ - pkg-config - llvmPackages.libclang - ]; - cargoToml = fromTOML (builtins.readFile ./Cargo.toml); commonArgs = { @@ -112,8 +79,7 @@ strictDeps = true; - # essentia-rs's essentia_sys links TensorFlow by default; virittaa does - # not need it. (Also set in .cargo/config.toml for non-Nix builds.) + # essentia_sys links TensorFlow unless this is 0. USE_TENSORFLOW = "0"; LIBCLANG_PATH = "${pkgs.llvmPackages.libclang.lib}/lib"; @@ -121,27 +87,21 @@ "-isystem ${pkgs.llvmPackages.libclang.lib}/lib/clang/${pkgs.llvmPackages.libclang.version}/include" + pkgs.lib.optionalString (!pkgs.stdenv.isDarwin) " -isystem ${pkgs.glibc.dev}/include"; - nativeBuildInputs = baseNativeBuildInputs; + nativeBuildInputs = [ + pkgs.pkg-config + pkgs.llvmPackages.libclang + ]; - # essentia-codegen's build script is linked against libessentia and - # then run by cargo to introspect the algorithm registry. rustc emits - # `-L native=...` but no RPATH, so the loader cannot find the .so: - # "error while loading shared libraries: libessentia.so". The upstream - # fix is for essentia_sys's build.rs to emit - # `cargo:rustc-link-arg=-Wl,-rpath,` for each probed library, - # which would also make the final binary self-contained and retire the - # wrapper below. + # essentia-codegen's build script links libessentia and is then run. + # Retired once essentia_sys emits a `-Wl,-rpath` link arg per probe. preBuild = '' export LD_LIBRARY_PATH="${pkgs.lib.makeLibraryPath runtimeLibs}''${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" ''; + # essentia_sys pkg-config probes. Listed explicitly rather than via + # essentia's propagation, from the attrs essentia.nix uses. buildInputs = [ - # essentia-rs's essentia_sys probes each of these via pkg-config at - # build time and links them into virittaa. List them explicitly - # (rather than relying on essentia's propagation) so the probes - # always resolve. These are the same nixpkgs attrs essentia.nix - # builds against, so the store paths match libessentia's. essentia pkgs.eigen pkgs.ffmpeg @@ -150,28 +110,21 @@ pkgs.libsamplerate pkgs.chromaprint pkgs.libyaml - # taglib's pkg-config Libs pulls in -lz. pkgs.zlib - # libsqlite3-sys links the system sqlite (no `bundled` feature). pkgs.sqlite ] ++ pkgs.lib.optionals pkgs.stdenv.isDarwin [pkgs.libiconv]; }; - # Build everything in one pass. The usual buildDepsOnly split is skipped - # because essentia-rs generates and compiles code from libessentia at - # build time, which does not survive crane's dummy-source dependency - # build. + # One pass: essentia-rs generates and compiles code from libessentia at + # build time, which does not survive crane's dummy-source dep build. virittaa = craneLib.buildPackage (commonArgs // { cargoArtifacts = null; nativeBuildInputs = commonArgs.nativeBuildInputs ++ [pkgs.makeWrapper]; postInstall = '' - # The build-time failure above shows rustc does not add an RPATH - # for these, so the installed binary needs the same help. Verify - # with `patchelf --print-rpath result/bin/.virittaa-wrapped`; - # if the rpath ever covers them, drop this and makeWrapper -- - # LD_LIBRARY_PATH leaks into anything virittaa execs. + # rustc emits no RPATH for these. Drop once + # `patchelf --print-rpath result/bin/.virittaa-wrapped` covers them. wrapProgram $out/bin/virittaa \ --prefix LD_LIBRARY_PATH : "${pkgs.lib.makeLibraryPath runtimeLibs}" ''; @@ -182,11 +135,7 @@ inherit virittaa essentia; }; - # `nix flake check` only. Note that clippy gets no dependency cache - # (cargoArtifacts = null, see above), so it is a second full build of - # the tree -- expensive, but opt-in. A cargoTest check is deliberately - # absent: cleanCargoSource strips non-Rust files, so any test fixture - # would go missing. + # `nix flake check` only: clippy is a second full build of the tree. checks = { inherit virittaa; fmt = craneLib.cargoFmt {inherit src;}; blob - /dev/null blob + dd0ca873a6aea4567ea5550797bacaeab260efbf (mode 755) --- /dev/null +++ hack/void-build.rb @@ -0,0 +1,473 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# Build virittaa, and the libessentia it links against, in a Void Linux chroot. +# Run as root. +# +# Essentia is not in the Void repos, and virittaa needs it as a library (headers, +# libessentia.so, essentia.pc) rather than the prebuilt extractor binary. Same +# commit and patch as nix/essentia.nix, so both builds line up. +# +# Output is a relocatable bundle: the binary plus its private libraries under an +# $ORIGIN rpath, runnable from anywhere without LD_LIBRARY_PATH. Fully static is +# not an option -- Void ships no static ffmpeg/taglib/chromaprint/fftw, and a +# static glibc breaks NSS. + +require "etc" +require "fileutils" +require "open3" +require "optparse" + +class Build + VOID_REPO = "https://repo-default.voidlinux.org/current" + + # Kept in sync with nix/essentia.nix. + ESSENTIA_REV = "b9fa6cb674ca43dfb94d28d293aeda441c6745db" + ESSENTIA_PATCH = "nix/essentia-network-thread-local.patch" + + # The git dependency in Cargo.toml; cloned unless -e gives a local checkout. + ESSENTIA_RS_URL = "ssh://anonymous@mtmn.name/essentia-rs.git" + + # ffmpeg6-devel rather than ffmpeg-devel, which is still 4.4.x; either + # provides libswresample, but essentia and virittaa must link the same one. + # eigen is headers-only (no -devel package). zlib-devel is needed because + # taglib's pkg-config Libs pulls in -lz. + DEPS = %w[ + base-devel patch python3 pkg-config + eigen fftw-devel ffmpeg6-devel libsamplerate-devel taglib-devel + chromaprint-devel libyaml-devel zlib-devel + rust cargo git clang sqlite-devel patchelf + ].freeze + + # Only what cargo needs: the working directory may also hold a music library. + SOURCE_ITEMS = %w[Cargo.toml Cargo.lock src .cargo].freeze + + # Provided by the host's glibc and loader; bundling them breaks the tree. + SYSTEM_LIBS = %w[ + linux-vdso.so.1 ld-linux-x86-64.so.2 libc.so.6 libm.so.6 + libpthread.so.0 libdl.so.2 librt.so.1 libresolv.so.2 + ].freeze + + VFS = { + "proc" => ["-t", "proc", "proc"], + "sys" => ["-t", "sysfs", "sys"], + "dev" => ["--bind", "/dev"] + }.freeze + + def initialize(argv) + @chroot = "/var/chroot/virittaa-build" + @source = Dir.pwd + @rev = ESSENTIA_REV + @install = true + @essentia_only = false + @delete = false + @quiet = false + option_parser.parse!(argv) + validate + end + + def run + bootstrap + mount_vfs + install_deps + build_essentia + return if @essentia_only + + stage_virittaa + build_virittaa + bundle + package + install_local if @install + ensure + unmount_vfs + if @delete + msg("removing #{@chroot}") + FileUtils.rm_rf(@chroot) + end + end + + private + + def option_parser + OptionParser.new do |o| + o.banner = "Usage: #{File.basename($PROGRAM_NAME)} [options]" + o.on("-c", "--chroot DIR", "chroot directory (default: #{@chroot})") { @chroot = it } + o.on("-s", "--source DIR", "virittaa source (default: current directory)") { @source = it } + o.on("-e", "--essentia-rs DIR", "local essentia-rs checkout (default: clone #{ESSENTIA_RS_URL})") { @essentia_rs = it } + o.on("-o", "--output DIR", "tarball output directory (default: invoking user's home)") { @output = it } + o.on("-r", "--rev REV", "essentia git revision (default: #{ESSENTIA_REV[0, 12]})") { @rev = it } + o.on("--essentia-src DIR", "build this essentia tree instead of fetching REV") { @essentia_src = it } + o.on("--essentia-only", "build libessentia, emit a tarball, and stop") { @essentia_only = true } + o.on("-n", "--no-install", "build only, do not install into ~/.local") { @install = false } + o.on("-d", "--delete", "delete the chroot when done (default: keep)") { @delete = true } + o.on("-q", "--quiet", "only report errors") { @quiet = true } + o.on("-h", "--help", "show this message") do + puts(o) + exit + end + end + end + + def validate + die("must run as root") unless Process.uid.zero? + + @source = File.expand_path(@source) + @patch = File.join(@source, ESSENTIA_PATCH) + die("#{ESSENTIA_PATCH} not found under #{@source}") unless File.file?(@patch) + + unless @essentia_only + die("no Cargo.toml in #{@source}") unless File.file?(File.join(@source, "Cargo.toml")) + if @essentia_rs + @essentia_rs = File.expand_path(@essentia_rs) + unless File.file?(File.join(@essentia_rs, "essentia_sys", "build.rs")) + die("#{@essentia_rs} does not look like an essentia-rs checkout") + end + end + end + + if @essentia_src + @essentia_src = File.expand_path(@essentia_src) + die("#{@essentia_src} does not look like essentia") unless File.directory?(File.join(@essentia_src, "src/essentia")) + end + + invoking_user = ENV["SUDO_USER"] || ENV["DOAS_USER"] + if invoking_user + @user = begin + Etc.getpwnam(invoking_user) + rescue ArgumentError + die("cannot resolve user #{invoking_user}") + end + end + # Nothing to install into without an invoking user; root's home is not it. + @install = false if @user.nil? + @output = File.expand_path(@output || @user&.dir || "~") + + @essentia_root = File.join(@chroot, "src/essentia") + @staging = File.join(@chroot, "staging") + @build_root = File.join(@chroot, "src/virittaa") + @dist = File.join(@chroot, "dist/virittaa") + @binary = File.join(@build_root, "target/release/virittaa") + end + + # --- plumbing ------------------------------------------------------------- + + def msg(text) + puts(text) unless @quiet + end + + def die(text) + warn("#{File.basename($PROGRAM_NAME)}: #{text}") + exit(1) + end + + def sh(*cmd) + msg(cmd.join(" ")) + _out, err, status = Open3.capture3(*cmd) + die("#{cmd.first} failed: #{err.strip}") unless status.success? + end + + # Via a file rather than a quoted -c string, so snippets can use quotes, + # $(...) and newlines freely. + def chroot_sh(label, script) + path = File.join(@chroot, "tmp/step.sh") + FileUtils.mkdir_p(File.dirname(path)) + File.write(path, "set -e\n#{script}") + msg(label) + die("#{label} failed") unless system("chroot", @chroot, "/bin/bash", "/tmp/step.sh") + end + + def chroot_capture(*cmd) + out, _err, status = Open3.capture3("chroot", @chroot, *cmd) + die("#{cmd.join(" ")} failed in chroot") unless status.success? + out + end + + # A moved pattern is fatal: skipping it yields a build that breaks much later. + def rewrite(path, from, to, label) + die("missing #{path}") unless File.file?(path) + + text = File.read(path) + return if text.include?(to) + die("#{path}: pattern not found: #{from}") unless text.include?(from) + + File.write(path, text.sub(from, to)) + msg(label) + end + + # `items` if given, otherwise everything but build output and git data. + def copy_tree(src, dst, items: nil) + FileUtils.mkdir_p(dst) + entries = items || (Dir.children(src) - ["target", ".git"]) + entries.each do |item| + path = File.join(src, item) + FileUtils.cp_r(path, dst, preserve: true) if File.exist?(path) + end + end + + def owned_by_user(path, recursive: false) + return unless @user + + if recursive + FileUtils.chown_R(@user.uid, @user.gid, path) + else + File.lchown(@user.uid, @user.gid, path) + end + rescue SystemCallError => e + msg("chown #{path} failed: #{e}") + end + + # --- chroot --------------------------------------------------------------- + + def bootstrap + copy_xbps_keys + return if File.executable?(File.join(@chroot, "bin/bash")) + + msg("bootstrapping chroot at #{@chroot}") + system("xbps-install", "-Syu", "-R", VOID_REPO, "-r", @chroot, "base-voidstrap") + die("bootstrap failed") unless File.executable?(File.join(@chroot, "bin/bash")) + copy_xbps_keys + end + + def copy_xbps_keys + keys = File.join(@chroot, "var/db/xbps/keys") + FileUtils.mkdir_p(keys) + FileUtils.cp(Dir.glob("/var/db/xbps/keys/*"), keys) + FileUtils.mkdir_p(File.join(@chroot, "etc")) + FileUtils.cp("/etc/resolv.conf", File.join(@chroot, "etc/resolv.conf")) + end + + def mount_vfs + VFS.each do |name, opts| + dst = File.join(@chroot, name) + FileUtils.mkdir_p(dst) + next if system("mountpoint", "-q", dst, out: File::NULL, err: File::NULL) + + system("mount", *opts, dst) + end + end + + def unmount_vfs + VFS.keys.reverse_each { system("umount", File.join(@chroot, it), err: File::NULL) } + end + + def install_deps + chroot_sh("installing build dependencies", <<~SH) + xbps-install -Syu + xbps-install -Sy #{DEPS.join(" ")} + SH + end + + # --- essentia ------------------------------------------------------------- + + def build_essentia + stage_essentia + FileUtils.mkdir_p(File.join(@chroot, "tmp")) + FileUtils.cp(@patch, File.join(@chroot, "tmp/essentia.patch")) + FileUtils.rm_rf(@staging) + + chroot_sh("building essentia (this takes a while)", <<~SH) + export HOME=/tmp + cd /src/essentia + if patch -p1 -R -f -s --dry-run -i /tmp/essentia.patch >/dev/null 2>&1; then + echo "essentia: already patched" + else + patch -p1 -i /tmp/essentia.patch + fi + python3 waf configure --prefix=/usr + python3 waf -j$(nproc) + python3 waf install --destdir=/staging + SH + + lib = File.join(@staging, "usr/lib/libessentia.so") + die("libessentia.so not found at #{lib}") unless File.file?(lib) + + # essentia.pc lists only per-subdirectory include paths, so the + # `#include ` the essentia-rs bridge emits does not resolve. + rewrite( + File.join(@staging, "usr/lib/pkgconfig/essentia.pc"), + "Cflags: -I${includedir}/essentia", + "Cflags: -I${includedir} -I${includedir}/essentia", + "patched essentia.pc" + ) + + package_essentia if @essentia_only + + chroot_sh("installing essentia into the chroot", <<~SH) + tar -C /staging -cf - . | tar -C / -xf - + ldconfig + echo "essentia $(pkg-config --modversion essentia) ok" + SH + end + + # On the host, so the chroot needs neither curl nor ca-certificates. + def stage_essentia + FileUtils.rm_rf(@essentia_root) + FileUtils.mkdir_p(File.dirname(@essentia_root)) + + if @essentia_src + msg("copying #{@essentia_src}") + FileUtils.cp_r(@essentia_src, @essentia_root, preserve: true) + return + end + + tarball = File.join(@chroot, "tmp/essentia.tar.gz") + FileUtils.mkdir_p(File.dirname(tarball)) + sh("curl", "-fsSL", "-o", tarball, "https://github.com/MTG/essentia/archive/#{@rev}.tar.gz") + FileUtils.mkdir_p(@essentia_root) + sh("tar", "xzf", tarball, "-C", @essentia_root, "--strip-components=1") + FileUtils.rm_f(tarball) + end + + def package_essentia + out = File.join(@output, "essentia-#{@rev[0, 12]}-#{Etc.uname[:machine]}.tar.gz") + sh("tar", "czf", out, "-C", @staging, ".") + owned_by_user(out) + msg("tarball: #{out} (install with: tar xzf #{out} -C /)") + end + + # --- virittaa ------------------------------------------------------------- + + # Re-staged from scratch every run; target/ is carried across so rebuilds in + # a persistent chroot stay incremental. + def stage_virittaa + target = File.join(@build_root, "target") + cache = File.join(@chroot, "src/target-cache") + if File.directory?(target) + FileUtils.rm_rf(cache) + FileUtils.mv(target, cache) + end + + FileUtils.rm_rf(@build_root) + FileUtils.mkdir_p(@build_root) + FileUtils.mv(cache, target) if File.directory?(cache) + + msg("staging virittaa source") + copy_tree(@source, @build_root, items: SOURCE_ITEMS) + + essentia_rs = @essentia_rs || clone_essentia_rs + msg("staging essentia-rs from #{essentia_rs}") + copy_tree(essentia_rs, File.join(@build_root, "vendor-essentia-rs")) + + # The chroot has no ssh access to the remote. + rewrite( + File.join(@build_root, "Cargo.toml"), + %(essentia = { git = "#{ESSENTIA_RS_URL}" }), + %(essentia = { path = "vendor-essentia-rs/essentia" }), + "pointed essentia at the staged checkout" + ) + unpin_lock + + # essentia_sys appends "eigen3" to every include path eigen3.pc reports, + # but Void's already ends in /eigen3. Same fix the flake applies. + rewrite( + File.join(@build_root, "vendor-essentia-rs/essentia_sys/build.rs"), + 'if library.name == "eigen3" {', + 'if library.name == "eigen3" && !include_path.join("Eigen").is_dir() {', + "patched essentia_sys eigen include path" + ) + end + + # Yields a path-resolved lock, matching the rewrite above, so the build stays + # --locked. + def unpin_lock + lock = File.join(@build_root, "Cargo.lock") + text = File.read(lock) + unpinned = text.gsub(/^source = "git\+#{Regexp.escape(ESSENTIA_RS_URL)}.*\n/o, "") + return if unpinned == text + + File.write(lock, unpinned) + msg("unpinned essentia-rs git source in Cargo.lock") + end + + # On the host, so the chroot needs no ssh setup. Kept beside the build root, + # which is wiped every run, and refreshed in place when already present. + def clone_essentia_rs + checkout = File.join(@chroot, "src/essentia-rs") + if File.directory?(File.join(checkout, ".git")) + msg("updating essentia-rs") + sh("git", "-C", checkout, "fetch", "--depth", "1", "origin", "HEAD") + sh("git", "-C", checkout, "reset", "--hard", "FETCH_HEAD") + else + FileUtils.rm_rf(checkout) + FileUtils.mkdir_p(File.dirname(checkout)) + sh("git", "clone", "--depth", "1", ESSENTIA_RS_URL, checkout) + end + checkout + end + + def build_virittaa + chroot_sh("building virittaa (this takes a while)", <<~SH) + export HOME=/tmp + export CARGO_HOME=/tmp/cargo + export USE_TENSORFLOW=0 + export LIBCLANG_PATH=/usr/lib + cd /src/virittaa + cargo build --release --locked + SH + + die("virittaa binary not found") unless File.file?(@binary) + end + + # $ORIGIN resolves against the real path of the binary, so the tree survives + # being moved or symlinked into ~/.local/bin. + def bundle + FileUtils.rm_rf(@dist) + FileUtils.mkdir_p(File.join(@dist, "bin")) + FileUtils.mkdir_p(File.join(@dist, "lib")) + FileUtils.cp(@binary, File.join(@dist, "bin/virittaa")) + + ldd = chroot_capture("ldd", "/src/virittaa/target/release/virittaa") + missing = ldd.lines.filter_map { it[/^\s*(\S+)\s+=>\s+not found/, 1] } + die("unresolved libraries: #{missing.join(", ")}") unless missing.empty? + + libs = ldd.lines + .filter_map { it[%r{=>\s+(/\S+)}, 1] } + .uniq + .reject { SYSTEM_LIBS.include?(File.basename(it)) } + .map { File.join(@chroot, it.delete_prefix("/")) } + .select { File.file?(it) } + die("ldd resolved no libraries") if libs.empty? + + FileUtils.cp(libs, File.join(@dist, "lib")) + msg("bundled #{libs.length} libraries") + + chroot_sh("setting rpaths", <<~SH) + patchelf --set-rpath '$ORIGIN/../lib' /dist/virittaa/bin/virittaa + for so in /dist/virittaa/lib/*.so*; do + patchelf --set-rpath '$ORIGIN' "$so" 2>/dev/null || true + done + /dist/virittaa/bin/virittaa --help > /dev/null + echo "in-chroot smoke test ok" + SH + end + + def package + version = File.read(File.join(@source, "Cargo.toml"))[/^version\s*=\s*"([^"]+)"/, 1] || "0" + @tarball = File.join(@output, "virittaa-#{version}-#{Etc.uname[:machine]}.tar.gz") + sh("tar", "czf", @tarball, "-C", File.dirname(@dist), "virittaa") + owned_by_user(@tarball) + msg("tarball: #{@tarball}") + end + + # Running it outside the chroot is what the $ORIGIN rpath is for. + def install_local + prefix = File.join(@user.dir, ".local") + target = File.join(prefix, "lib/virittaa") + link = File.join(prefix, "bin/virittaa") + + FileUtils.mkdir_p([File.dirname(target), File.dirname(link)]) + FileUtils.rm_rf(target) + sh("tar", "xzf", @tarball, "-C", File.dirname(target)) + FileUtils.rm_f(link) + FileUtils.ln_s(File.join(target, "bin/virittaa"), link) + owned_by_user(target, recursive: true) + owned_by_user(link) + + out, _err, status = Open3.capture3(link, "--help") + die("installed binary does not run") unless status.success? + msg(out.lines.first.to_s.strip) + msg("installed: #{link} (add ~/.local/bin to PATH if it is not there already)") + end +end + +Build.new(ARGV).run blob - a93f4fb12587ed5af0fb43b75ba3f92df01cd7ae (mode 644) blob + /dev/null --- nix/BUILD_NOTES.md +++ /dev/null @@ -1,176 +0,0 @@ -# virittaa + Essentia (essentia-rs) — Nix build notes - -Working notes for getting `nix build` to build everything correctly. Pick up -from "Current status" below. - -## Goal - -virittaa's BPM/key analysis is delegated to Essentia via the **essentia-rs** -bindings crate (the user's sibling repo at `/home/miro/src/sr.ht/essentia-rs`, -git `git.sr.ht:~mtmn/essentia-rs`, branch `main`). The old hand-written C++ shim -was removed. `nix build` must build: - -1. `libessentia` (C++) from source — `nix/essentia.nix`. -2. virittaa (Rust), which compiles the essentia-rs bindings against that lib. - -## How the pieces fit - -- `Cargo.toml` has `essentia = { path = "../essentia-rs/essentia" }` (sibling). -- essentia-rs's `essentia` crate has a **build-dependency `essentia-codegen` - that runs libessentia at build time** (introspects algorithm names/params to - generate the typed Rust API). So libessentia must be present *and loadable* - while the `essentia` crate compiles. -- essentia-rs's `essentia_sys` `build.rs` probes these via pkg-config and links - them: `essentia eigen3 yaml-0.1 fftw3f taglib samplerate libchromaprint - libavformat libswresample libavcodec libavutil` — plus **tensorflow unless - `USE_TENSORFLOW=0`**. Note it links **swresample** (modern ffmpeg), not - avresample; that is why essentia itself must be built from master, not from - the v2.1_beta5 tag (see below). - -## Flake structure (flake.nix) - -- New input `essentia-rs` (`git+file:///home/miro/src/sr.ht/essentia-rs`, - `flake = false`). Relative `path:../essentia-rs` is rejected by Nix ("points - outside store path"); the sr.ht https remote 404s (not published). Switch to - `git+https://git.sr.ht/~mtmn/essentia-rs` once pushed. -- Removed `naersk` input and all `mkCross` cross-compile targets — you cannot - cross-compile libessentia + ffmpeg/taglib/etc to win/mac this way. -- `src` is assembled by a `runCommandLocal`: copies `cleanCargoSource ./.`, then - copies the essentia-rs source into `nix-vendor-essentia-rs/` and rewrites the - Cargo.toml path `../essentia-rs/essentia` → `nix-vendor-essentia-rs/essentia`. - essentia-rs is copied **raw** (not cleanCargoSource) because its C++ bridge - files (`essentia_sys/bridge/*.cpp,*.h`) would be stripped by the cargo filter. -- The same step patches `essentia_sys/build.rs`: it appends `eigen3` to every - include path `eigen3.pc` reports, which is correct on Debian (`-I/usr/include`) - but wrong on nixpkgs (`-I$out/include/eigen3` already), yielding - `include/eigen3/eigen3` and `fatal error: unsupported/Eigen/CXX11/Tensor: No - such file or directory`. **This is an essentia-rs bug** — fix it there - (`if library.name == "eigen3" && !include_path.join("Eigen").is_dir()`), then - drop the patch here; `--replace-fail` will flag it once upstream changes. -- `USE_TENSORFLOW = "0"` set in `commonArgs` (also in `.cargo/config.toml` for - non-Nix builds). -- Single-shot build: `cargoArtifacts = null` (no crane buildDepsOnly). The dep - split is skipped because crane's dummy-source build would strip essentia-rs's - C++ files and can't run the codegen. Cost: every rebuild recompiles the whole - dep tree. -- `buildInputs` lists the full pkg-config set explicitly (essentia, eigen, - ffmpeg, fftwFloat, taglib, libsamplerate, chromaprint, libyaml, zlib) so the - essentia_sys probes always resolve, rather than relying on essentia's - propagatedBuildInputs (multi-output `.dev` propagation is unreliable). -- `essentia` is exposed as a package output: `nix build .#essentia`. -- No `cmake` in nativeBuildInputs — nothing in the lockfile needs it (zstd-sys - and libsqlite3-sys both use `cc`). No `glibc.dev` either; the only thing that - wanted it was bindgen, which gets it via `BINDGEN_EXTRA_CLANG_ARGS`. -- `checks` adds `fmt` and `clippy`. `clippy` is a second uncached full build, - so it only runs under `nix flake check`, never `nix build`. - -## nix/essentia.nix - -- essentia is pinned to a **master commit** (`b9fa6cb`, 2.1-beta6-dev, - 2026-05-20), not the `v2.1_beta5` tag. beta5 is a dead end: it needs - `libavresample` (removed in ffmpeg 5, so it forces `ffmpeg_4`) and bundles - waf 2.0.10, which needs Python ≤ 3.11 (`import imp`) *and* source patching - for the `'rU'` open mode Python 3.11 removed. Master uses `libswresample` - and bundles waf 2.0.25, which runs unmodified on Python 3.14. -- Consequently: plain `ffmpeg`, plain `taglib` (2.x satisfies the wscript's - `taglib >= 1.9` check, and the API essentia uses — `FileRef`, `PropertyMap`, - `StringList` — survived the taglib 2.0 break), plain `python3`. No pins. -- `zlib` is required: taglib's pkg-config `Libs` pulls in `-lz` and nothing - else in the closure provides it. Symptom if it regresses: all 288 objects - compile, then `ld.bfd: cannot find -lz` while linking `libessentia.so`. -- propagatedBuildInputs: eigen fftwFloat ffmpeg libsamplerate taglib - chromaprint libyaml zlib. -- `eigen` is the one *mandatory* pkg-config check on master (`eigen3 >= 3.3.4`); - every other `check_cfg` in `src/wscript` is `mandatory=False`, so a missing - dep there silently drops algorithms instead of failing the build. Worth - reading the "CONFIGURATION SUMMARY" block in the log — MonoLoader, - MetadataReader, Resample etc. disappear if ffmpeg/taglib/samplerate go - missing, and you only find out when the Rust codegen can't find them. -- installPhase patches essentia.pc Cflags to add `-I${includedir}` - (essentia.pc.in still lists only the per-subdirectory include paths). -- `postPatch` makes `scheduler::Network::lastCreated` `thread_local`. Upstream - it is one process-global pointer, written by every Network ctor/dtor and read - by every `Network::run()`. MonoLoader, RhythmExtractor2013 and KeyExtractor - each build an inner Network, so analysing with `-j >1` corrupts it: warning - spam ("No network created..."), interleaved output, then `free(): double free - detected in tcache`. The same hunk adds the missing `return` after that - warning, which otherwise falls straight into a null dereference. Essentia - supports one-instance-per-thread otherwise (FFTW plans and the devnull/copy - singletons are ForcedMutex-guarded); this global is the outlier. **Report - upstream.** `src/audio_processing.rs` documents that it depends on this. -- To move the pin: `nix hash path --sri --type sha256 `, or set - `hash = lib.fakeHash` and read the correct value out of the build error. - -## Building outside Nix (Void Linux) - -`void-build-essentia.rb` (repo root, run as root) builds libessentia in a -throwaway Void chroot and emits an installable tarball. It deliberately mirrors -this derivation: same pinned commit, same two `Network::lastCreated` patches, -same `essentia.pc` Cflags fixup. Keep the two in sync — `src/audio_processing.rs` -runs Essentia's `compute` unlocked, which is only safe against a patched build. - -`void-build-virittaa.rb` drives that script, installs the resulting essentia -into the same chroot, then builds virittaa and emits a relocatable bundle -(binary + private libs, `$ORIGIN/../lib` rpath), optionally unpacking it into -the invoking user's `~/.local` and smoke-testing it. It applies the same -essentia_sys eigen include-path patch the flake does. - -A fully static binary is not on the table: Void ships no static -ffmpeg/taglib/chromaprint/fftw, and a statically linked glibc breaks NSS at -runtime. The `$ORIGIN` bundle is the portable substitute, and unlike the flake's -`wrapProgram` it needs no `LD_LIBRARY_PATH`. - -Void specifics: `ffmpeg6-devel`, not `ffmpeg-devel` (still 4.4.x); `eigen` is -headers-only with no `-devel`; `zlib-devel` is needed because taglib's -pkg-config `Libs` pulls in `-lz`; `rust`/`cargo` are 1.97, new enough for -edition 2024. Whichever ffmpeg virittaa links must be the one essentia was -built against. - -## Current status - -- [x] Rust integration done (`src/audio_processing.rs` uses - MonoLoader → RhythmExtractor2013(multifeature) → KeyExtractor(EDMA)). -- [x] flake restructured (path-dep vendoring, single-shot, USE_TENSORFLOW=0). -- [x] `nix build .#essentia` — builds clean on master against ffmpeg 8.1, - taglib 2.2.1, eigen 3.4.1, Python 3.14. -- [x] The C++ bridge compiles (after the eigen include fix above). -- [x] essentia-codegen's build script needs `LD_LIBRARY_PATH` — rustc emits - `-L native=...` but no RPATH, so *running* the build script fails with - "libessentia.so: cannot open shared object file", and once that resolves, - "libstdc++.so.6: ..." (the cxx bridge). Both come from the shared - `essentiaLibs` list, exported in `preBuild` and reused by the runtime - wrapper; `stdenv.cc.cc.lib` is in it for libstdc++. - Upstream fix: have essentia_sys's build.rs emit - `cargo:rustc-link-arg=-Wl,-rpath,` per probed library; that would - retire both the preBuild export and the wrapper. -- [x] `nix build .#virittaa` — builds and links. The codegen ran against - master's libessentia, so the module paths and enum variants in - audio_processing.rs (`essentia::algorithm::{input_output,rhythm,tonal}`, - `RhythmExtractor2013Method::Multifeature`, `KeyExtractorProfileType::Edma`) - are confirmed correct. `sqlite` had to be added to buildInputs: - libsqlite3-sys builds unbundled and links the system library. -- [ ] Verify `-j >1` analysis is now clean end to end (this is what surfaced the - `lastCreated` bug above). -- [ ] Drop the `LD_LIBRARY_PATH` wrapper in `postInstall` once the built binary - is confirmed to carry the right RPATH (`patchelf --print-rpath - result/bin/.virittaa-wrapped`). - -## Reproduce / continue - -```sh -cd /home/miro/src/sr.ht/virittaa -git add -A # flake (git) only sees tracked/staged files! -nix build .#essentia -L # libessentia alone -nix build .#virittaa -L # the real test -``` - -Note: because the flake is `git+file://…/virittaa`, **untracked files are -invisible to the build** — `git add -A` after every new file (e.g. this one, -`nix/essentia.nix`) before building. Modified tracked files are picked up from -the working tree (Nix warns "dirty"). - -## Assumptions still to verify on first successful compile - -- Generated algorithm module paths and the two parameter enum variants (above). - If wrong, they're one-line path fixes in `src/audio_processing.rs`. -- essentia-codegen can load libessentia at build time under Nix. blob - /dev/null blob + 159f900beb9542b2ad02a131bf7dc7a39be269c1 (mode 644) --- /dev/null +++ nix/essentia-network-thread-local.patch @@ -0,0 +1,51 @@ +Make scheduler::Network::lastCreated thread-local. + +lastCreated is a process-global pointer, written by every Network constructor +and destructor and read by every Network::run() via +printNetworkBufferFillState(). MonoLoader, RhythmExtractor2013 and KeyExtractor +each build an inner Network, so analysing files on more than one thread makes +threads clobber each other's pointer: warning spam, interleaved output, then a +use-after-free ("double free detected in tcache"). It is only a debug +convenience, so make it per-thread; Essentia otherwise supports +one-instance-per-thread use (FFTW plan creation and the devnull/copy singletons +are ForcedMutex-guarded). + +The second hunk fixes the null dereference on the line right after the warning +that says the pointer is null. + +Not yet submitted upstream. + +diff --git a/src/essentia/scheduler/network.cpp b/src/essentia/scheduler/network.cpp +index 7046731d..64bf94fc 100644 +--- a/src/essentia/scheduler/network.cpp ++++ b/src/essentia/scheduler/network.cpp +@@ -164,7 +164,7 @@ vector NetworkNode::addVisibleDependencies(mapprintBufferFillState(); +diff --git a/src/essentia/scheduler/network.h b/src/essentia/scheduler/network.h +index c8712b2c..acc5e0d2 100644 +--- a/src/essentia/scheduler/network.h ++++ b/src/essentia/scheduler/network.h +@@ -200,7 +200,7 @@ class Network { + * Last instance of Network created, 0 if it has been deleted or if + * no network has been created yet. + */ +- static Network* lastCreated; ++ static thread_local Network* lastCreated; + + protected: + bool _takeOwnership; blob - b7cc49ef2fa3e553dd0c918446c23983bebb430b blob + 3d8c2637a8bc83962ddd3f516f95cb0f1e218415 --- nix/essentia.nix +++ nix/essentia.nix @@ -1,10 +1,6 @@ -# Pinned to a master commit (2.1-beta6-dev) rather than the last tag, v2.1_beta5. -# beta5 requires libavresample (removed in ffmpeg 5) and bundles waf 2.0.10, -# which needs Python <= 3.11 (`import imp`) and patching for the removed 'rU' -# open mode. Master resamples with libswresample instead -- matching what -# essentia-rs's essentia_sys probes via pkg-config -- and bundles waf 2.0.25, -# which runs unmodified on current Python. Both pins (ffmpeg_4, taglib_1) and -# the whole waf patching dance are gone as a result. +# Master commit, not the v2.1_beta5 tag: beta5 needs libavresample (gone since +# ffmpeg 5) and bundles a waf that will not run on Python >= 3.12. Master +# resamples with libswresample, which is what essentia-rs probes for. { lib, stdenv, @@ -20,7 +16,7 @@ libyaml, zlib, }: -stdenv.mkDerivation (finalAttrs: { +stdenv.mkDerivation { pname = "essentia"; version = "2.1-beta6-dev-unstable-2026-05-20"; @@ -31,29 +27,9 @@ stdenv.mkDerivation (finalAttrs: { hash = "sha256-8z7SZN6tO/3iB2bOzPnz1en0NjbgGJcTjkj02PzvdVs="; }; - # `scheduler::Network::lastCreated` is a process-global pointer, assigned by - # every Network constructor and cleared by every destructor, and read on every - # `Network::run()` via printNetworkBufferFillState(). Since MonoLoader, - # RhythmExtractor2013 and KeyExtractor each build an inner Network, analysing - # files on more than one thread makes threads clobber each other's pointer: - # "No network created, or last created network has been deleted..." spam, - # interleaved output, then a use-after-free ("double free detected in tcache"). - # - # It is purely a debug convenience, so make it per-thread. Essentia otherwise - # supports one-instance-per-thread use (it guards FFTW plan creation and the - # devnull/copy singletons with ForcedMutex); this global is the outlier. - # The second hunk fixes the null dereference on the line right after the - # warning that says the pointer is null. Both are worth reporting upstream. - postPatch = '' - substituteInPlace src/essentia/scheduler/network.h \ - --replace-fail 'static Network* lastCreated;' \ - 'static thread_local Network* lastCreated;' - substituteInPlace src/essentia/scheduler/network.cpp \ - --replace-fail 'Network* Network::lastCreated = 0;' \ - 'thread_local Network* Network::lastCreated = 0;' \ - --replace-fail 'E_WARNING("No network created, or last created network has been deleted...");' \ - 'E_WARNING("No network created, or last created network has been deleted..."); return;' - ''; + # Required for multi-threaded analysis; see the patch header. + # hack/void-build.rb applies the same one. + patches = [./essentia-network-thread-local.patch]; strictDeps = true; @@ -62,12 +38,10 @@ stdenv.mkDerivation (finalAttrs: { pkg-config ]; - # Propagated so anything linking libessentia.so also picks up its runtime - # dependencies (the .so records them as NEEDED). + # libessentia.so records these as NEEDED. propagatedBuildInputs = [ - # The only hard dependency on master (`eigen3 >= 3.3.4`, mandatory); every - # other check_cfg in src/wscript is mandatory=False. Header-only, but - # propagated because essentia's own headers include it. + # The only mandatory check in src/wscript; the rest are mandatory=False and + # silently drop algorithms when unmet. eigen fftwFloat ffmpeg @@ -75,15 +49,12 @@ stdenv.mkDerivation (finalAttrs: { taglib chromaprint libyaml - # taglib's pkg-config Libs pulls in -lz; nothing else in the closure - # provides it, so libessentia.so fails to link without it. - zlib + zlib # taglib's pkg-config Libs pulls in -lz ]; configurePhase = '' runHook preConfigure - # waf caches its unpacked waflib under $HOME. - export HOME=$TMPDIR + export HOME=$TMPDIR # waf unpacks its waflib cache there python3 waf configure --prefix=$out runHook postConfigure ''; @@ -97,9 +68,8 @@ stdenv.mkDerivation (finalAttrs: { installPhase = '' runHook preInstall python3 waf install - # essentia.pc.in lists only the per-subdirectory include paths, so - # `#include ` (what the essentia-rs bridge writes) does not - # resolve. Add the parent include dir. + # essentia.pc.in lists only per-subdirectory include paths, so the + # `#include ` the essentia-rs bridge emits does not resolve. substituteInPlace $out/lib/pkgconfig/essentia.pc \ --replace-fail 'Cflags: -I''${includedir}/essentia' \ 'Cflags: -I''${includedir} -I''${includedir}/essentia' @@ -112,4 +82,4 @@ stdenv.mkDerivation (finalAttrs: { license = lib.licenses.agpl3Plus; platforms = lib.platforms.linux; }; -}) +} blob - f63663e8f03aabdbc0f8102cb7a463969528e342 blob + 205dcc00c13a83ab58adb38155b1270b8092e263 --- src/audio_processing.rs +++ src/audio_processing.rs @@ -1,42 +1,34 @@ //! BPM and key detection, backed by Essentia (https://github.com/MTG/essentia) //! via the `essentia` bindings crate (essentia-rs). //! -//! The pipeline mirrors Essentia's own extractors: `MonoLoader` decodes and -//! downmixes to mono at 44.1 kHz, `RhythmExtractor2013` (multifeature) estimates -//! tempo, and `KeyExtractor` (EDMA profile) estimates key. Unlike the previous -//! aubio/Krumhansl pipeline, the reported BPM is Essentia's true tempo estimate: -//! there is no magic offset and no octave-folding into a fixed range. +//! `MonoLoader` decodes and downmixes to mono at 44.1 kHz, `RhythmExtractor2013` +//! (multifeature) estimates tempo, `KeyExtractor` (EDMA) estimates key. The BPM +//! is Essentia's estimate: no offset, no octave-folding into a range. use std::path::Path; use std::sync::{LazyLock, Mutex}; use anyhow::{Context, Result}; -use essentia::Essentia; -use essentia::GetFromDataContainer; use essentia::algorithm::input_output::MonoLoader; use essentia::algorithm::rhythm::{RhythmExtractor2013, RhythmExtractor2013Method}; use essentia::algorithm::tonal::{KeyExtractor, KeyExtractorProfileType}; +use essentia::{Essentia, GetFromDataContainer}; /// Sample rate Essentia's extractors expect; also what `MonoLoader` resamples to. const SAMPLE_RATE: f32 = 44100.0; -/// Process-wide Essentia handle. `Essentia::new()` runs `essentia::init()` on the -/// first call and reference-counts a global lifecycle, so keeping one alive here -/// keeps the library initialized for the life of the process. +/// `Essentia::new()` reference-counts a global lifecycle, so one live handle +/// keeps the library initialized for the process. static ESSENTIA: LazyLock = LazyLock::new(Essentia::new); -/// Essentia's `AlgorithmFactory` is not safe for concurrent `create`, and the -/// composite extractors below also instantiate inner algorithms through it while -/// configuring. virittaa analyzes files in parallel (rayon), so we serialize the -/// create+configure phase; the heavy `compute` calls run unlocked. +/// `AlgorithmFactory` is not safe for concurrent `create`, and the composite +/// extractors instantiate inner algorithms through it while configuring, so the +/// create+configure phase is serialized; `compute` runs unlocked. /// -/// Running `compute` unlocked additionally requires an Essentia whose -/// `scheduler::Network::lastCreated` is `thread_local`. Upstream it is a single -/// process-global pointer that every inner Network's constructor and destructor -/// writes and every `Network::run()` reads, so concurrent analysis corrupts it -/// and aborts with a double free. `nix/essentia.nix` patches it; when building -/// against a distro Essentia, either apply the same patch or hold this lock -/// across the whole function. +/// Unlocked `compute` requires an Essentia whose `scheduler::Network::lastCreated` +/// is `thread_local` (`nix/essentia-network-thread-local.patch`); upstream it is +/// a process-global that concurrent analysis corrupts into a double free. +/// Against an unpatched build, hold this lock across the whole function. static FACTORY_LOCK: Mutex<()> = Mutex::new(()); /// Result of analysing a single track. @@ -59,12 +51,10 @@ pub struct Analysis { /// Returns an error if the path is not valid UTF-8 or if Essentia fails to /// decode or analyze the file. pub fn analyze_file(path: &Path) -> Result { - // Essentia's MonoLoader takes the path as a string parameter. let path_str = path .to_str() .with_context(|| format!("file path is not valid UTF-8: {}", path.display()))?; - // Decode + downmix to mono at 44.1 kHz. let mut loader = { let _guard = lock_factory(); ESSENTIA @@ -80,7 +70,6 @@ pub fn analyze_file(path: &Path) -> Result { .audio() .get(); - // Tempo via the multifeature beat-tracker ensemble. let mut rhythm = { let _guard = lock_factory(); ESSENTIA @@ -95,7 +84,6 @@ pub fn analyze_file(path: &Path) -> Result { let bpm = rhythm_result.bpm().get(); let bpm_confidence = rhythm_result.confidence().get(); - // Key via the EDMA (electronic dance music) profile. let mut key = { let _guard = lock_factory(); ESSENTIA @@ -119,8 +107,8 @@ pub fn analyze_file(path: &Path) -> Result { }) } -/// Acquire the factory lock, recovering from a poisoned mutex (a panic in another -/// analysis thread must not wedge every remaining file). +/// Recovers from poisoning: a panic in one analysis thread must not wedge every +/// remaining file. fn lock_factory() -> std::sync::MutexGuard<'static, ()> { FACTORY_LOCK.lock().unwrap_or_else(|e| e.into_inner()) } blob - 3b22eaf2e1d12ceb691e3a77aadea3874493efc9 blob + 8be736f201e7b934f721c5579ceb1d9e3e82a710 --- src/db.rs +++ src/db.rs @@ -16,19 +16,17 @@ pub struct TrackReport { pub timestamp: Option, /// Essentia's tempo confidence. `None` for records written before this field /// existed, for Mixxx imports, and for tracks read from existing file tags. - #[serde(default)] pub bpm_confidence: Option, /// Essentia's key correlation strength. `None` for the same reasons. - #[serde(default)] pub key_strength: Option, } /// `TrackReport` as it was stored before the confidence fields were added. /// -/// postcard is not self-describing: a struct is just its fields back to back, -/// with no names or count, so `#[serde(default)]` cannot rescue a record that -/// is simply missing the trailing bytes. Decoding an old record as the current -/// struct hits EOF, so `deserialize_report` falls back to this shape. +/// postcard is not self-describing -- a struct is its fields back to back, with +/// no names or count -- so `#[serde(default)]` cannot rescue a record missing +/// the trailing bytes; decoding one as the current struct hits EOF. +/// `deserialize_report` falls back to this shape instead. #[derive(Deserialize)] struct LegacyTrackReport { artist: Option, @@ -141,17 +139,16 @@ fn serialize_report(report: &TrackReport) -> Result Result { - match postcard::from_bytes::(bytes) { - Ok(report) => Ok(report), - Err(current_err) => postcard::from_bytes::(bytes) - .map(TrackReport::from) - .map_err(|_| current_err) - .context("Failed to deserialize track report"), - } + postcard::from_bytes::(bytes) + .or_else(|err| { + postcard::from_bytes::(bytes) + .map(TrackReport::from) + .map_err(|_| err) + }) + .context("Failed to deserialize track report") } fn effective_limit(limit: usize) -> usize { blob - d2da6d6da80c6acdf70e8fb443c728036a4ae173 blob + cc8195f28191a8df91f2dfdac9b3acded7ba6c97 --- src/file_io.rs +++ src/file_io.rs @@ -18,45 +18,18 @@ pub struct FileMetadata { pub track: Option, } -fn build_track_info( - path: &Path, - metadata: FileMetadata, - bpm: f32, - key: String, - bpm_confidence: Option, - key_strength: Option, -) -> TrackInfo { - TrackInfo { - path: path.to_path_buf(), - artist: metadata.artist, - track: metadata.track, - bpm, - key, - bpm_confidence, - key_strength, - } -} - -/// Render Essentia's tempo confidence for the console. -/// -/// `RhythmExtractor2013`'s multifeature method reports roughly 0..5.3, where -/// Essentia treats below 1.5 as unreliable and above 3.5 as strong. Flagging the -/// weak estimates is the point: octave errors and ambiguous tracks land there. -fn format_confidence(bpm_confidence: Option, key_strength: Option) -> String { - let Some(conf) = bpm_confidence else { - return String::new(); - }; - let verdict = if conf < 1.5 { +/// `RhythmExtractor2013`'s multifeature method reports roughly 0..5.3; Essentia +/// treats below 1.5 as unreliable and above 3.5 as strong. Octave errors and +/// ambiguous material land in the low band. +fn format_confidence(bpm_confidence: f32, key_strength: f32) -> String { + let verdict = if bpm_confidence < 1.5 { "weak" - } else if conf < 3.5 { + } else if bpm_confidence < 3.5 { "fair" } else { "strong" }; - key_strength.map_or_else( - || format!(" (tempo {conf:.2} {verdict})"), - |strength| format!(" (tempo {conf:.2} {verdict}, key {strength:.2})"), - ) + format!(" (tempo {bpm_confidence:.2} {verdict}, key {key_strength:.2})") } /// Process a single file, detecting BPM and key. @@ -69,38 +42,43 @@ pub fn process_file(path: &Path, write_tags: bool, for if !force && let Some((bpm, key)) = existing_tags { println!( - "File: {}, BPM: {:.1}, Key: {} (existing tags)", - path.display(), + "File: {}, BPM: {bpm:.1}, Key: {key} (existing tags)", + path.display() + ); + return Ok(TrackInfo { + path: path.to_path_buf(), + artist: metadata.artist, + track: metadata.track, bpm, - key - ); - return Ok(build_track_info(path, metadata, bpm, key, None, None)); + key, + bpm_confidence: None, + key_strength: None, + }); } let analysis = analyze_file(path).map_err(|e| e.to_string())?; let bpm = analysis.bpm; let key = analysis.key; - let bpm_confidence = Some(analysis.bpm_confidence); - let key_strength = Some(analysis.key_strength); println!( "File: {}, BPM: {bpm:.1}, Key: {key}{}", path.display(), - format_confidence(bpm_confidence, key_strength) + format_confidence(analysis.bpm_confidence, analysis.key_strength) ); if write_tags && let Err(e) = write_metadata(path, bpm, &key) { eprintln!("Error writing metadata for {}: {e}", path.display()); } - Ok(build_track_info( - path, - metadata, + Ok(TrackInfo { + path: path.to_path_buf(), + artist: metadata.artist, + track: metadata.track, bpm, key, - bpm_confidence, - key_strength, - )) + bpm_confidence: Some(analysis.bpm_confidence), + key_strength: Some(analysis.key_strength), + }) } fn read_tags(path: &Path) -> (FileMetadata, Option<(f32, String)>) { @@ -322,40 +300,11 @@ mod tests { use super::*; #[test] - fn build_track_info_carries_metadata_and_analysis() { - let metadata = FileMetadata { - artist: Some("Aphex Twin".to_string()), - track: Some("Xtal".to_string()), - }; - let info = build_track_info( - Path::new("/music/xtal.flac"), - metadata, - 120.5, - "A Minor".to_string(), - Some(3.9), - Some(0.81), - ); - - assert_eq!(info.path, Path::new("/music/xtal.flac")); - assert_eq!(info.artist.as_deref(), Some("Aphex Twin")); - assert_eq!(info.track.as_deref(), Some("Xtal")); - assert!((info.bpm - 120.5).abs() < f32::EPSILON); - assert_eq!(info.key, "A Minor"); - assert_eq!(info.bpm_confidence, Some(3.9)); - assert_eq!(info.key_strength, Some(0.81)); - } - - #[test] - fn format_confidence_bands_and_omits_when_absent() { - assert_eq!(format_confidence(None, None), ""); - assert_eq!(format_confidence(None, Some(0.9)), ""); - assert_eq!(format_confidence(Some(0.4), None), " (tempo 0.40 weak)"); + fn format_confidence_bands_the_tempo_estimate() { + assert_eq!(format_confidence(0.4, 0.5), " (tempo 0.40 weak, key 0.50)"); + assert_eq!(format_confidence(2.0, 0.5), " (tempo 2.00 fair, key 0.50)"); assert_eq!( - format_confidence(Some(2.0), Some(0.5)), - " (tempo 2.00 fair, key 0.50)" - ); - assert_eq!( - format_confidence(Some(4.2), Some(0.77)), + format_confidence(4.2, 0.77), " (tempo 4.20 strong, key 0.77)" ); } blob - 2365721f10755df280269d44b72f4926bf08b124 (mode 755) blob + /dev/null --- void-build-essentia.rb +++ /dev/null @@ -1,343 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -# Build libessentia in a throwaway Void Linux chroot and emit an installable -# tarball. -# -# Essentia is not in the Void repos, and virittaa needs it as a *library* -# (headers + libessentia.so + essentia.pc), not the prebuilt extractor binary. -# This mirrors what nix/essentia.nix does, so a Void build lines up with the -# Nix one: same pinned master commit, same two upstream patches, same -# essentia.pc fixup. - -require "fileutils" -require "open3" -require "etc" - -class Builder - REPO = "https://repo-default.voidlinux.org/current" - - # The last tag, v2.1_beta5, needs libavresample (gone since ffmpeg 5) and - # bundles waf 2.0.10, which cannot run on Python 3.12+. Master resamples with - # libswresample and bundles waf 2.0.25, which runs fine on Void's python3. - REV = "b9fa6cb674ca43dfb94d28d293aeda441c6745db" - - # ffmpeg6-devel rather than ffmpeg-devel: the latter is still 4.4.x. Either - # provides libswresample, but whatever is chosen here must match what virittaa - # itself links against. eigen is headers-only (no -devel package). zlib-devel - # is required because taglib's pkg-config Libs pulls in -lz. - DEPS = %w[ - base-devel - python3 - pkg-config - eigen - fftw-devel - ffmpeg6-devel - libsamplerate-devel - taglib-devel - chromaprint-devel - libyaml-devel - zlib-devel - ] - .freeze - - # `scheduler::Network::lastCreated` is a process-global pointer written by - # every Network constructor/destructor and read on every `Network::run()`. - # MonoLoader, RhythmExtractor2013 and KeyExtractor each build an inner - # Network, so analysing on more than one thread corrupts it and aborts with a - # double free. It is only a debug convenience, so make it per-thread. The - # third hunk fixes the null dereference on the line after the warning that - # says the pointer is null. Both are worth reporting upstream. - PATCHES = [ - [ - "src/essentia/scheduler/network.h", - "static Network* lastCreated;", - "static thread_local Network* lastCreated;" - ], - [ - "src/essentia/scheduler/network.cpp", - "Network* Network::lastCreated = 0;", - "thread_local Network* Network::lastCreated = 0;" - ], - [ - "src/essentia/scheduler/network.cpp", - 'E_WARNING("No network created, or last created network has been deleted...");', - 'E_WARNING("No network created, or last created network has been deleted..."); return;' - ] - ].freeze - - def initialize(argv) - @outdir = nil - @chroot = "/var/chroot/essentia-build" - @rev = REV - @src = nil - @prefix = "/usr" - @delete = false - @quiet = false - parse(argv) - validate - end - - def run - bootstrap - mount_vfs - prep_chroot - install_deps - stage_source - patch_source - build - extract - ensure - unmount_vfs - remove_chroot if @delete - msg("done.") - end - - private - - def parse(argv) - i = 0 - while i < argv.length - case argv[i] - when "-o" - @outdir = argv[i += 1] - when "-c" - @chroot = argv[i += 1] - when "-r" - @rev = argv[i += 1] - when "-s" - @src = argv[i += 1] - when "-p" - @prefix = argv[i += 1] - when "-d" - @delete = true - when "-q" - @quiet = true - when "-h", "--help" - usage(code: 0) - else - usage - end - - i += 1 - end - end - - def usage_lines - [ - "Usage: #{File.basename($0)} [-o DIR] [-c DIR] [-r REV] [-s DIR] [-p PREFIX] [-d] [-q]", - " -o DIR output directory (default: invoking user home)", - " -c DIR chroot directory (default: /var/chroot/essentia-build)", - " -r REV essentia git revision (default: #{REV[0, 12]})", - " -s DIR build this source tree instead of fetching REV", - " -p PREFIX install prefix inside the tarball (default: /usr)", - " -d delete chroot after build (default: keep)", - " -q quiet" - ] - end - - def usage(code: 1) - usage_lines.each { |l| puts(l) } - exit(code) - end - - def validate - unless Process.uid.zero? - usage_lines.each { |l| puts(l) } - die("must run as root") - end - - if @src - @src = File.expand_path(@src) - die("no wscript in #{@src}") unless File.file?(File.join(@src, "wscript")) - die("#{@src} does not look like essentia") unless File.directory?(File.join(@src, "src/essentia")) - end - - inv = ENV["SUDO_USER"] || ENV["DOAS_USER"] - if inv - @pw = begin - Etc.getpwnam(inv) - rescue - nil - end - die("cannot resolve user #{inv}") unless @pw - @outdir ||= @pw.dir - else - @outdir ||= File.expand_path("~") - end - - @build_root = File.join(@chroot, "src/essentia") - @stage = File.join(@chroot, "staging") - end - - def msg(s) - puts(s) unless @quiet - end - - def die(s) - warn("#{File.basename($0)}: #{s}") - exit(1) - end - - def run_cmd(cmd, fatal: true) - msg(cmd) - _o, e, st = Open3.capture3(cmd) - die("#{cmd}: #{e.strip}") if !st.success? && fatal - st.success? - end - - # Run a shell snippet inside the chroot by dropping it in a file and invoking - # bash on it, rather than interpolating into a quoted -c string. Keeps the - # build scripts free to use quotes, $(...) and newlines. - def chroot_sh(label, script) - path = File.join(@chroot, "tmp/step.sh") - FileUtils.mkdir_p(File.dirname(path)) - File.write(path, "set -e\n#{script}") - msg(label) - die("#{label} failed") unless system("chroot", @chroot, "/bin/bash", "/tmp/step.sh") - end - - def bootstrap - FileUtils.mkdir_p(@chroot) - return if File.executable?(File.join(@chroot, "bin/bash")) - - msg("bootstrapping chroot...") - copy_keys - run_cmd("xbps-install -Syu -R #{REPO} -r #{@chroot} base-voidstrap", fatal: false) - die("bootstrap failed") unless File.executable?(File.join(@chroot, "bin/bash")) - msg("bootstrap ok") - end - - def copy_keys - kdir = File.join(@chroot, "var/db/xbps/keys") - FileUtils.mkdir_p(kdir) - Dir.glob("/var/db/xbps/keys/*").each { |f| FileUtils.cp(f, kdir) } - end - - def mount_vfs - mounts = { - File.join(@chroot, "proc") => ["-t", "proc", "proc"], - File.join(@chroot, "sys") => ["-t", "sysfs", "sys"], - File.join(@chroot, "dev") => ["--bind", "/dev"] - } - mounts.each do |dst, opts| - FileUtils.mkdir_p(dst) - next if system("mountpoint -q #{dst} 2>/dev/null") - system("mount #{opts.join(" ")} #{dst}") - end - end - - def unmount_vfs - %w[dev sys proc].each { |n| system("umount #{File.join(@chroot, n)} 2>/dev/null") } - end - - def prep_chroot - FileUtils.cp("/etc/resolv.conf", File.join(@chroot, "etc/resolv.conf")) - copy_keys - end - - def install_deps - chroot_sh("installing deps", <<~SH) - xbps-install -Syu - xbps-install -Sy #{DEPS.join(" ")} - SH - end - - # Fetched on the host rather than in the chroot, so the chroot needs neither - # curl nor ca-certificates. - def stage_source - FileUtils.rm_rf(@build_root) - FileUtils.mkdir_p(File.dirname(@build_root)) - - if @src - msg("copying #{@src}") - system("cp", "-a", @src, @build_root) || die("copy failed") - return - end - - tarball = File.join(@chroot, "tmp/essentia.tar.gz") - FileUtils.mkdir_p(File.dirname(tarball)) - url = "https://github.com/MTG/essentia/archive/#{@rev}.tar.gz" - run_cmd("curl -fsSL -o #{tarball} #{url}") - - FileUtils.mkdir_p(@build_root) - run_cmd("tar xzf #{tarball} -C #{@build_root} --strip-components=1") - FileUtils.rm_f(tarball) - end - - # Applied on the host, on the copied tree. Idempotent, and fatal if a pattern - # has moved -- silently skipping would produce a library that crashes only - # under concurrency, which is exactly the failure this prevents. - def patch_source - PATCHES.each do |rel, from, to| - path = File.join(@build_root, rel) - die("missing #{rel}") unless File.file?(path) - - txt = File.read(path) - if txt.include?(to) - msg("already patched: #{rel}") - next - end - die("#{rel}: pattern not found: #{from}") unless txt.include?(from) - - File.write(path, txt.sub(from, to)) - msg("patched #{rel}") - end - end - - def build - FileUtils.rm_rf(@stage) - chroot_sh("building essentia", <<~SH) - export HOME=/tmp - cd /src/essentia - python3 waf configure --prefix=#{@prefix} - python3 waf -j$(nproc) - python3 waf install --destdir=/staging - SH - - lib = File.join(@stage, @prefix.delete_prefix("/"), "lib/libessentia.so") - die("libessentia.so not found at #{lib}") unless File.file?(lib) - - fix_pkgconfig - end - - # essentia.pc lists only the per-subdirectory include paths, so the - # `#include ` that essentia-rs's bridge emits does not resolve. - # Add the parent include dir, exactly as nix/essentia.nix does. - def fix_pkgconfig - pc = File.join(@stage, @prefix.delete_prefix("/"), "lib/pkgconfig/essentia.pc") - die("essentia.pc not found at #{pc}") unless File.file?(pc) - - txt = File.read(pc) - from = "Cflags: -I${includedir}/essentia" - to = "Cflags: -I${includedir} -I${includedir}/essentia" - return if txt.include?(to) - die("essentia.pc: Cflags line not found") unless txt.include?(from) - - File.write(pc, txt.sub(from, to)) - msg("patched essentia.pc") - end - - def extract - out = File.join(@outdir, "essentia-#{@rev[0, 12]}-#{`uname -m`.strip}.tar.gz") - run_cmd("tar czf #{out} -C #{@stage} .") - - if @pw - begin - File.chown(@pw.uid, @pw.gid, out) - rescue => e - msg("chown failed: #{e}") - end - end - - msg("tarball: #{out}") - msg("install with: tar xzf #{out} -C /") - end - - def remove_chroot - msg("removing #{@chroot}...") - FileUtils.rm_rf(@chroot) - end -end - -Builder.new(ARGV).run blob - 473cf300aa7e7a51d4a25f264be3ce77c32edfd1 (mode 755) blob + /dev/null --- void-build-virittaa.rb +++ /dev/null @@ -1,437 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -# Build virittaa in a throwaway Void Linux chroot and install a relocatable -# bundle into the invoking user's ~/.local. -# -# Drives void-build-essentia.rb first (libessentia is not in the Void repos), -# installs the result into the chroot, then builds virittaa against it and -# bundles the binary with its non-glibc shared libraries. -# -# On "static": a fully static binary is not achievable here. Void ships no -# static ffmpeg/taglib/chromaprint/fftw, and statically linking glibc breaks -# NSS at runtime. What this produces instead is a *relocatable* bundle -- the -# binary plus its private libs, wired up with an $ORIGIN rpath -- which runs -# from any directory without LD_LIBRARY_PATH and without installing essentia -# system-wide. - -require "fileutils" -require "open3" -require "etc" - -class Builder - ESSENTIA_SCRIPT = "void-build-essentia.rb" - - DEPS = %w[ - rust - cargo - git - clang - sqlite-devel - patchelf - ] - .freeze - - # Only what cargo needs. The virittaa working directory may also hold a music - # library, which must not be copied into the chroot. - SRC_ITEMS = %w[Cargo.toml Cargo.lock src .cargo].freeze - - # Provided by the host's glibc and loader; bundling these is what breaks a - # relocatable tree rather than what saves it. - SYSTEM_LIBS = %w[ - linux-vdso.so.1 - ld-linux-x86-64.so.2 - libc.so.6 - libm.so.6 - libpthread.so.0 - libdl.so.2 - librt.so.1 - libresolv.so.2 - ].freeze - - def initialize(argv) - @outdir = nil - @chroot = "/var/chroot/virittaa-build" - @src = Dir.pwd - @essentia_rs = nil - @rev = nil - @install = true - @delete = false - @quiet = false - parse(argv) - validate - end - - def run - build_essentia - # The essentia builder unmounts the vfs on its way out; cargo and rustc - # need /dev/null and /proc, so put them back. - mount_vfs - install_essentia - install_deps - stage_source - build - bundle - package - install_local if @install - ensure - unmount_vfs - remove_chroot if @delete - msg("done.") - end - - private - - def parse(argv) - i = 0 - while i < argv.length - case argv[i] - when "-o" - @outdir = argv[i += 1] - when "-c" - @chroot = argv[i += 1] - when "-s" - @src = argv[i += 1] - when "-e" - @essentia_rs = argv[i += 1] - when "-r" - @rev = argv[i += 1] - when "-n" - @install = false - when "-d" - @delete = true - when "-q" - @quiet = true - when "-h", "--help" - usage(code: 0) - else - usage - end - - i += 1 - end - end - - def usage_lines - [ - "Usage: #{File.basename($0)} [-o DIR] [-c DIR] [-s DIR] [-e DIR] [-r REV] [-n] [-d] [-q]", - " -o DIR output directory for the tarball (default: invoking user home)", - " -c DIR chroot directory (default: /var/chroot/virittaa-build)", - " -s DIR virittaa source (default: current directory)", - " -e DIR essentia-rs source (default: SRC/../essentia-rs)", - " -r REV essentia git revision (passed to #{ESSENTIA_SCRIPT})", - " -n build only, do not install into ~/.local", - " -d delete chroot after build (default: keep)", - " -q quiet" - ] - end - - def usage(code: 1) - usage_lines.each { |l| puts(l) } - exit(code) - end - - def validate - unless Process.uid.zero? - usage_lines.each { |l| puts(l) } - die("must run as root") - end - - @src = File.expand_path(@src) - die("no Cargo.toml in #{@src}") unless File.file?(File.join(@src, "Cargo.toml")) - - @essentia_script = File.join(@src, ESSENTIA_SCRIPT) - die("#{ESSENTIA_SCRIPT} not found next to the source") unless File.file?(@essentia_script) - - @essentia_rs = File.expand_path(@essentia_rs || File.join(@src, "..", "essentia-rs")) - unless File.file?(File.join(@essentia_rs, "essentia_sys", "build.rs")) - die("no essentia-rs checkout at #{@essentia_rs} (use -e DIR)") - end - - inv = ENV["SUDO_USER"] || ENV["DOAS_USER"] - if inv - @pw = begin - Etc.getpwnam(inv) - rescue - nil - end - die("cannot resolve user #{inv}") unless @pw - @outdir ||= @pw.dir - else - @outdir ||= File.expand_path("~") - @install = false if @install && !@pw - end - - @build_root = File.join(@chroot, "src/virittaa") - @dist = File.join(@chroot, "dist/virittaa") - @binary = File.join(@build_root, "target/release/virittaa") - end - - def msg(s) - puts(s) unless @quiet - end - - def die(s) - warn("#{File.basename($0)}: #{s}") - exit(1) - end - - def run_cmd(cmd, fatal: true) - msg(cmd) - _o, e, st = Open3.capture3(cmd) - die("#{cmd}: #{e.strip}") if !st.success? && fatal - st.success? - end - - def chroot_sh(label, script) - path = File.join(@chroot, "tmp/step.sh") - FileUtils.mkdir_p(File.dirname(path)) - File.write(path, "set -e\n#{script}") - msg(label) - die("#{label} failed") unless system("chroot", @chroot, "/bin/bash", "/tmp/step.sh") - end - - def chroot_capture(script) - path = File.join(@chroot, "tmp/query.sh") - FileUtils.mkdir_p(File.dirname(path)) - File.write(path, "set -e\n#{script}") - out, _e, st = Open3.capture3("chroot", @chroot, "/bin/bash", "/tmp/query.sh") - die("chroot query failed") unless st.success? - out - end - - # Copy a tree without dragging in build output (or, for the virittaa source, - # anything not on the allowlist). - def copy_tree(src, dst, items: nil, excludes: %w[./target]) - FileUtils.mkdir_p(dst) - if items - items.each do |item| - path = File.join(src, item) - next unless File.exist?(path) - system("cp", "-a", path, dst) || die("copy #{item} failed") - end - else - args = excludes.map { |e| "--exclude=#{e}" }.join(" ") - run_cmd("tar -C #{src} -cf - #{args} . | tar -C #{dst} -xf -") - end - end - - # The essentia builder owns bootstrapping the chroot; it keeps it by default, - # and its tarball lands in the chroot's own tmp. -d is deliberately never - # passed down: this script shares that chroot and tears it down itself. - def build_essentia - FileUtils.mkdir_p(File.join(@chroot, "tmp")) - cmd = ["ruby", @essentia_script, "-c", @chroot, "-o", File.join(@chroot, "tmp")] - cmd += ["-r", @rev] if @rev - cmd << "-q" if @quiet - - msg(cmd.join(" ")) - die("essentia build failed") unless system(*cmd) - end - - def install_essentia - tarballs = Dir.glob(File.join(@chroot, "tmp/essentia-*.tar.gz")) - die("no essentia tarball produced") if tarballs.empty? - inner = "/tmp/#{File.basename(tarballs.max_by { |f| File.mtime(f) })}" - - chroot_sh("installing essentia into chroot", <<~SH) - tar xzf #{inner} -C / - ldconfig - pkg-config --exists essentia - echo "essentia $(pkg-config --modversion essentia) ok" - SH - end - - def install_deps - chroot_sh("installing rust toolchain and build deps", <<~SH) - xbps-install -Sy #{DEPS.join(" ")} - SH - end - - # The source is re-staged from scratch every run, but the chroot now persists - # by default, so carry target/ across so a rebuild is incremental rather than - # recompiling the whole dependency tree. - def stage_source - target = File.join(@build_root, "target") - cache = File.join(@chroot, "src/target-cache") - if File.directory?(target) - FileUtils.rm_rf(cache) - FileUtils.mv(target, cache) - msg("preserving cargo target/ for incremental rebuild") - end - - FileUtils.rm_rf(@build_root) - FileUtils.mkdir_p(@build_root) - FileUtils.mv(cache, target) if File.directory?(cache) - - msg("staging virittaa source") - copy_tree(@src, @build_root, items: SRC_ITEMS) - - vendor = File.join(@build_root, "vendor-essentia-rs") - msg("staging essentia-rs from #{@essentia_rs}") - copy_tree(@essentia_rs, vendor) - - rewrite_path_dep - patch_eigen_include - end - - # The sibling path dependency does not exist inside the chroot. - def rewrite_path_dep - manifest = File.join(@build_root, "Cargo.toml") - txt = File.read(manifest) - from = "../essentia-rs/essentia" - to = "vendor-essentia-rs/essentia" - return if txt.include?(to) - die("Cargo.toml: path dependency #{from} not found") unless txt.include?(from) - - File.write(manifest, txt.sub(from, to)) - msg("rewrote essentia path dependency") - end - - # essentia_sys's build.rs appends "eigen3" to every include path eigen3.pc - # reports. Void's eigen3.pc already points at .../include/eigen3, so the - # bridge would look in include/eigen3/eigen3 and fail on - # . Same fix the flake applies; drop this - # once essentia-rs handles both layouts. - def patch_eigen_include - path = File.join(@build_root, "vendor-essentia-rs/essentia_sys/build.rs") - die("missing vendored essentia_sys/build.rs") unless File.file?(path) - - txt = File.read(path) - from = 'if library.name == "eigen3" {' - to = 'if library.name == "eigen3" && !include_path.join("Eigen").is_dir() {' - return if txt.include?(to) - die("essentia_sys/build.rs: eigen guard not found") unless txt.include?(from) - - File.write(path, txt.sub(from, to)) - msg("patched essentia_sys eigen include path") - end - - def build - chroot_sh("building virittaa (this takes a while)", <<~SH) - export HOME=/tmp - export CARGO_HOME=/tmp/cargo - export USE_TENSORFLOW=0 - export LIBCLANG_PATH=/usr/lib - cd /src/virittaa - cargo build --release --locked - SH - - die("virittaa binary not found") unless File.file?(@binary) - end - - # Copy the binary plus every non-glibc shared library it resolves, then point - # its rpath at the private lib directory. $ORIGIN is resolved by the loader - # relative to the real path of the binary, so the tree can be moved or - # symlinked into ~/.local/bin without breaking. - def bundle - FileUtils.rm_rf(@dist) - FileUtils.mkdir_p(File.join(@dist, "bin")) - FileUtils.mkdir_p(File.join(@dist, "lib")) - - FileUtils.cp(@binary, File.join(@dist, "bin/virittaa")) - - ldd = chroot_capture("ldd /src/virittaa/target/release/virittaa") - - missing = ldd.lines.filter_map { |l| l[/^\s*(\S+)\s+=>\s+not found/, 1] } - die("unresolved libraries: #{missing.join(", ")}") unless missing.empty? - - libs = ldd - .lines - .filter_map { |l| l[%r{=>\s+(/\S+)}, 1] } - .uniq - .reject { |p| SYSTEM_LIBS.include?(File.basename(p)) } - - die("ldd resolved no libraries") if libs.empty? - - libs.each do |lib| - host_path = File.join(@chroot, lib.delete_prefix("/")) - next unless File.file?(host_path) - FileUtils.cp(host_path, File.join(@dist, "lib", File.basename(lib))) - end - msg("bundled #{libs.length} libraries") - - chroot_sh("setting rpaths", <<~SH) - patchelf --set-rpath '$ORIGIN/../lib' /dist/virittaa/bin/virittaa - for so in /dist/virittaa/lib/*.so*; do - patchelf --set-rpath '$ORIGIN' "$so" 2>/dev/null || true - done - /dist/virittaa/bin/virittaa --help > /dev/null - echo "in-chroot smoke test ok" - SH - end - - def package - @tarball = File.join(@outdir, "virittaa-#{version}-#{`uname -m`.strip}.tar.gz") - run_cmd("tar czf #{@tarball} -C #{File.dirname(@dist)} virittaa") - chown_to_user(@tarball) - msg("tarball: #{@tarball}") - end - - def version - File.read(File.join(@src, "Cargo.toml"))[/^version\s*=\s*"([^"]+)"/, 1] || "0" - end - - # Unpack into the invoking user's ~/.local and prove it runs outside the - # chroot, which is the whole point of the $ORIGIN rpath. - def install_local - target = File.join(@pw.dir, ".local/lib/virittaa") - bindir = File.join(@pw.dir, ".local/bin") - FileUtils.mkdir_p(File.dirname(target)) - FileUtils.mkdir_p(bindir) - FileUtils.rm_rf(target) - - run_cmd("tar xzf #{@tarball} -C #{File.dirname(target)}") - - link = File.join(bindir, "virittaa") - FileUtils.rm_f(link) - FileUtils.ln_s(File.join(target, "bin/virittaa"), link) - - chown_recursive(target) - chown_to_user(link) - - out, _e, st = Open3.capture3(link, "--help") - die("installed binary does not run") unless st.success? - msg(out.lines.first.to_s.strip) - msg("installed: #{link}") - msg("add ~/.local/bin to PATH if it is not there already") - end - - def chown_to_user(path) - return unless @pw - File.lchown(@pw.uid, @pw.gid, path) - rescue => e - msg("chown failed: #{e}") - end - - def chown_recursive(dir) - return unless @pw - FileUtils.chown_R(@pw.uid, @pw.gid, dir) - rescue => e - msg("chown failed: #{e}") - end - - def mount_vfs - mounts = { - File.join(@chroot, "proc") => ["-t", "proc", "proc"], - File.join(@chroot, "sys") => ["-t", "sysfs", "sys"], - File.join(@chroot, "dev") => ["--bind", "/dev"] - } - mounts.each do |dst, opts| - FileUtils.mkdir_p(dst) - next if system("mountpoint -q #{dst} 2>/dev/null") - system("mount #{opts.join(" ")} #{dst}") - end - end - - def unmount_vfs - %w[dev sys proc].each { |n| system("umount #{File.join(@chroot, n)} 2>/dev/null") } - end - - def remove_chroot - msg("removing #{@chroot}...") - FileUtils.rm_rf(@chroot) - end -end - -Builder.new(ARGV).run