Skip to content

Instantly share code, notes, and snippets.

@tinsukE

tinsukE/freebsd_13.2_py-paperless-ngx_patch.md

Last active June 20, 2024 09:26
Show Gist options
  • Save tinsukE/81898d120a13a84ee02db2e9ffdf055d to your computer and use it in GitHub Desktop.
Save tinsukE/81898d120a13a84ee02db2e9ffdf055d to your computer and use it in GitHub Desktop.
Download ZIP
Patches to install py-paperless-ngx 2.7.2 on FreeBSD 13.2
Raw
freebsd_13.2_py-paperless-ngx_patch.md

Patches to install py-paperless-ngx 2.7.2 on FreeBSD 13.2, which can come in handy if you can't upgrade, yet (ex: you run TrueNAS Core).

os.sched_getaffinity.patch

numpy and joblib need to be patched to avoid an exception when calling os.getaffinity():

OverflowError: could not allocate a large enough CPU set

This might be something needed for my system only, which is a FreeBSD 13.1 machine (TrueNAS CORE) hosting a FreeBSD 13.2 jail.

py-threadpoolctl.patch

py-threadpoolctl needs to be patched to work on FreeBSD <= 13.2:

Raw
install_paperless.sh
#!/bin/sh
set DEPS_LIST="py39-pip py39-bleach py39-concurrent-log-handler \
py39-dj42-django-cors-headers py39-dj42-django-filter py39-filelock \
py39-gunicorn py39-h2 py39-hiredis py39-httptools py39-langdetect \
py39-mysqlclient py39-ocrmypdf py39-pdftotext py39-psycopg2 \
py39-python-dotenv py39-python-gnupg py39-python-magic \
py39-setproctitle py39-sqlite3 py39-uvicorn py39-uvloop py39-watchfiles \
py39-websockets py39-whitenoise py39-whoosh py39-yaml \
gnupg liberation-fonts-ttf optipng tesseract unpaper \
blas openblas cmake gcc13 poppler-utils zbar ImageMagick7"
echo $DEPS_LIST | xargs pkg install -y
# 51.332u 8.035s 3:56.50 25.0% 3001+234k 3+184933io 1479pf+0w
portsnap fetch extract
# 59.524u 86.625s 2:39.78 91.4% 50+165k 1+202655io 40pf+0w
# Patch os.sched_getaffinity()
curl https://gist.githubusercontent.com/tinsukE/81898d120a13a84ee02db2e9ffdf055d/raw/cb2634c2d8e9db983ad3cf36e8d99c9674a9fc17/os.sched_getaffinity.patch > os.sched_getaffinity.patch
patch -ruN -V none -p0 < os.sched_getaffinity.patch
# Patch py-threadpoolctl for FreeBSD 13.2
curl https://gist.githubusercontent.com/tinsukE/81898d120a13a84ee02db2e9ffdf055d/raw/982b7ee866feb4da1451614c88d9ba57085a251d/py-threadpoolctl.patch > py-threadpoolctl.patch
patch -ruN -V none -p0 < py-threadpoolctl.patch
setenv BATCH yes
cd /usr/ports/deskutils/py-paperless-ngx
make install
# 2324.296u 166.586s 21:57.22 189.1% 50524+764k 4+429944io 79972pf+0w
echo $DEPS_LIST | xargs pkg set -A 1 -y
pkg autoremove -y
cd && rm -rf /usr/ports
Raw
os.sched_getaffinity.patch
diff -ruN /usr/ports/devel/py-joblib/files/patch-joblib_externals_loky_backend_context.py /usr/ports/devel/py-joblib.patched/files/patch-joblib_externals_loky_backend_context.py
--- /usr/ports/devel/py-joblib/files/patch-joblib_externals_loky_backend_context.py 2024-01-05 17:50:06.000000000 +0100
+++ /usr/ports/devel/py-joblib.patched/files/patch-joblib_externals_loky_backend_context.py 2024-04-18 10:34:21.271261845 +0200
@@ -1,5 +1,14 @@
--- joblib/externals/loky/backend/context.py.orig 2023-06-29 15:14:21 UTC
+++ joblib/externals/loky/backend/context.py
+@@ -186,7 +186,7 @@
+ if hasattr(os, "sched_getaffinity"):
+ try:
+ return len(os.sched_getaffinity(0))
+- except NotImplementedError:
++ except (NotImplementedError, OverflowError):
+ pass
+
+ # On PyPy and possibly other platforms, os.sched_getaffinity does not exist
@@ -245,6 +245,9 @@ def _count_physical_cores():
return physical_cores_cache, exception
diff -ruN /usr/ports/math/py-numpy/files/patch-numpy_distutils_misc__util.py /usr/ports/math/py-numpy.patched/files/patch-numpy_distutils_misc__util.py
--- /usr/ports/math/py-numpy/files/patch-numpy_distutils_misc__util.py 1970-01-01 01:00:00.000000000 +0100
+++ /usr/ports/math/py-numpy.patched/files/patch-numpy_distutils_misc__util.py 2024-04-18 10:22:39.684285126 +0200
@@ -0,0 +1,11 @@
+--- numpy/distutils/misc_util.py.orig 2024-04-18 08:20:32 UTC
++++ numpy/distutils/misc_util.py
+@@ -91,7 +91,7 @@ def get_num_build_jobs():
+ from numpy.distutils.core import get_distribution
+ try:
+ cpu_count = len(os.sched_getaffinity(0))
+- except AttributeError:
++ except (AttributeError, OverflowError):
+ cpu_count = multiprocessing.cpu_count()
+ cpu_count = min(cpu_count, 8)
+ envjobs = int(os.environ.get("NPY_NUM_BUILD_JOBS", cpu_count))
Raw
py-threadpoolctl.patch
diff -ruN /usr/ports/devel/py-threadpoolctl/Makefile /usr/ports/devel/py-threadpoolctl.patched/Makefile
--- /usr/ports/devel/py-threadpoolctl/Makefile 2024-04-17 19:37:46.592777841 +0200
+++ /usr/ports/devel/py-threadpoolctl.patched/Makefile 2024-04-17 19:09:48.140323452 +0200
@@ -20,8 +20,4 @@
.include <bsd.port.options.mk>
-.if ${OPSYS} == FreeBSD && ${OSVERSION} < 1302509
-IGNORE= does not run on FreeBSD 13.2 or earlier due to handling of dynamically loaded libc
-.endif
-
.include <bsd.port.mk>
diff -ruN /usr/ports/devel/py-threadpoolctl/files/patch-threadpoolctl.py /usr/ports/devel/py-threadpoolctl.patched/files/patch-threadpoolctl.py
--- /usr/ports/devel/py-threadpoolctl/files/patch-threadpoolctl.py 1970-01-01 01:00:00.000000000 +0100
+++ /usr/ports/devel/py-threadpoolctl.patched/files/patch-threadpoolctl.py 2024-04-17 19:37:34.856053076 +0200
@@ -0,0 +1,73 @@
+--- threadpoolctl.py.orig 2024-04-17 15:48:02 UTC
++++ threadpoolctl.py
+@@ -67,6 +67,7 @@ try:
+ except AttributeError:
+ _RTLD_NOLOAD = ctypes.DEFAULT_MODE
+
++_RTLD_DEFAULT=-2
+
+ class LibController(ABC):
+ """Abstract base class for the individual library controllers
+@@ -981,8 +982,62 @@ class ThreadpoolController:
+ self._find_libraries_with_enum_process_module_ex()
+ elif "pyodide" in sys.modules:
+ self._find_libraries_pyodide()
++ elif "freebsd" in sys.platform:
++ self._find_libraries_with_dl_iterate_phdr_freebsd()
+ else:
+ self._find_libraries_with_dl_iterate_phdr()
++
++ def _find_libraries_with_dl_iterate_phdr_freebsd(self):
++ """Loop through loaded libraries and return binders on supported ones
++
++ This function is expected to work on POSIX system only.
++ This code is adapted from code by Intel developer @anton-malakhov
++ available at https://github.com/IntelPython/smp
++
++ Copyright (c) 2017, Intel Corporation published under the BSD 3-Clause
++ license
++ """
++
++ parent = ctypes.CDLL(None)
++
++ dlopen = parent.dlopen
++ dlopen.restype = ctypes.c_void_p
++
++ dlsym = parent.dlsym
++ dlsym.restype = ctypes.c_void_p
++
++ dl_iterate_phdr_ptr = dlsym(_RTLD_DEFAULT, b'dl_iterate_phdr')
++
++ if dl_iterate_phdr_ptr == 0:
++ return []
++
++ # Callback function for `dl_iterate_phdr` which is called for every
++ # library loaded in the current process until it returns 1.
++ def match_library_callback(info, size, data):
++ # Get the path of the current library
++ filepath = info.contents.dlpi_name
++ if filepath:
++ filepath = filepath.decode("utf-8")
++
++ # Store the library controller if it is supported and selected
++ self._make_controller_from_path(filepath)
++ return 0
++
++ c_func_signature = ctypes.CFUNCTYPE(
++ ctypes.c_int, # Return type
++ ctypes.POINTER(_dl_phdr_info),
++ ctypes.c_size_t,
++ ctypes.c_char_p,
++ )
++ c_match_library_callback = c_func_signature(match_library_callback)
++
++ data = ctypes.c_char_p(b"")
++
++ dl_iterate_phdr_c_signature = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p)
++ dl_iterate_phdr_c_signature.argtypes = [ c_func_signature, ctypes.c_void_p ]
++ dl_iterate_phdr = dl_iterate_phdr_c_signature(dl_iterate_phdr_ptr)
++
++ dl_iterate_phdr(c_match_library_callback, data)
+
+ def _find_libraries_with_dl_iterate_phdr(self):
+ """Loop through loaded libraries and return binders on supported ones
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment