That actually bugged me so much that I made a script to use the manifest to blow away declared files and dirs

    PKGID="${1:-}"

    PKG_ROOT="$(pkgutil --pkg-info "$PKGID" \
        | awk -F: '/location:/{print $2}' \
        | sed -e 's,^[ \t]*,/,')"
    
    pkgutil --only-files --files "$PKGID" \
        | join_paths_0 "$PKG_ROOT" \
        | xargs -0 $dry_run rm -v
    pkgutil --only-dirs --files "$PKGID" \
        | sort -r \
        | join_paths_0 "$PKG_ROOT" \
        | xargs -0 $dry_run rmdir
    pkgutil --forget "$PKGID"

It still leaves detritus, but is better than "oh well, guess I'm stuck with Rando Thing 0.9beta forever"