Hi,
I would like to use PyMC in a remotely hosted web service on Render. I define a python environment in a requirements.txt file and then Render builds it remotely. Unfortunately, I have problems getting PyMC to work. When sampling from a model, I am getting the following error message:
You can find the C code in this temporary file: /tmp/pytensor_compilation_error_ajenws21
raise ImportError(
ImportError: Version check of the existing lazylinker compiled file. Looking for version 0.212, but found None. Extra debug information: force_compile=False, _need_reload=True
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/render/project/src/.venv/lib/python3.10/site-packages/pytensor/link/c/lazylinker_c.py", line 87, in <module>
raise ImportError(
ImportError: Version check of the existing lazylinker compiled file. Looking for version 0.212, but found None. Extra debug information: force_compile=False, _need_reload=True
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/render/project/src/.venv/bin/gunicorn", line 8, in <module>
sys.exit(run())
File "/opt/render/project/src/.venv/lib/python3.10/site-packages/gunicorn/app/wsgiapp.py", line 66, in run
WSGIApplication("%(prog)s [OPTIONS] [APP_MODULE]", prog=prog).run()
File "/opt/render/project/src/.venv/lib/python3.10/site-packages/gunicorn/app/base.py", line 235, in run
super().run()
File "/opt/render/project/src/.venv/lib/python3.10/site-packages/gunicorn/app/base.py", line 71, in run
Arbiter(self).run()
File "/opt/render/project/src/.venv/lib/python3.10/site-packages/gunicorn/arbiter.py", line 57, in __init__
self.setup(app)
File "/opt/render/project/src/.venv/lib/python3.10/site-packages/gunicorn/arbiter.py", line 117, in setup
self.app.wsgi()
File "/opt/render/project/src/.venv/lib/python3.10/site-packages/gunicorn/app/base.py", line 66, in wsgi
self.callable = self.load()
File "/opt/render/project/src/.venv/lib/python3.10/site-packages/gunicorn/app/wsgiapp.py", line 57, in load
return self.load_wsgiapp()
File "/opt/render/project/src/.venv/lib/python3.10/site-packages/gunicorn/app/wsgiapp.py", line 47, in load_wsgiapp
return util.import_app(self.app_uri)
File "/opt/render/project/src/.venv/lib/python3.10/site-packages/gunicorn/util.py", line 370, in import_app
mod = importlib.import_module(module)
File "/opt/render/project/python/Python-3.10.6/lib/python3.10/importlib/__init__.py", line 126, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 883, in exec_module
File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
File "/opt/render/project/src/fblv.py", line 547, in <module>
bpm.fit(tr_df)
File "/opt/render/project/src/fblv.py", line 309, in fit
approx = pm.fit(progressbar=False)
File "/opt/render/project/src/.venv/lib/python3.10/site-packages/pymc/variational/inference.py", line 759, in fit
inference = _select[method](model=model, **inf_kwargs)
File "/opt/render/project/src/.venv/lib/python3.10/site-packages/pymc/variational/inference.py", line 471, in __init__
super().__init__(MeanField(*args, **kwargs))
File "/opt/render/project/src/.venv/lib/python3.10/site-packages/pymc/variational/approximations.py", line 339, in __init__
super().__init__(groups, model=kwargs.get("model"))
File "/opt/render/project/src/.venv/lib/python3.10/site-packages/pymc/variational/opvi.py", line 1229, in __init__
rest.__init_group__(unseen_free_RVs)
File "/opt/render/project/src/.venv/lib/python3.10/site-packages/pytensor/configparser.py", line 44, in res
return f(*args, **kwargs)
File "/opt/render/project/src/.venv/lib/python3.10/site-packages/pymc/variational/approximations.py", line 72, in __init_group__
super().__init_group__(group)
File "/opt/render/project/src/.venv/lib/python3.10/site-packages/pytensor/configparser.py", line 44, in res
return f(*args, **kwargs)
File "/opt/render/project/src/.venv/lib/python3.10/site-packages/pymc/variational/opvi.py", line 853, in __init_group__
model_initial_point = self.model.initial_point(0)
File "/opt/render/project/src/.venv/lib/python3.10/site-packages/pymc/model/core.py", line 1094, in initial_point
fn = make_initial_point_fn(model=self, return_transformed=True)
File "/opt/render/project/src/.venv/lib/python3.10/site-packages/pymc/initial_point.py", line 152, in make_initial_point_fn
func = compile_pymc(inputs=[], outputs=initial_values, mode=pytensor.compile.mode.FAST_COMPILE)
File "/opt/render/project/src/.venv/lib/python3.10/site-packages/pymc/pytensorf.py", line 1039, in compile_pymc
pytensor_function = pytensor.function(
File "/opt/render/project/src/.venv/lib/python3.10/site-packages/pytensor/compile/function/__init__.py", line 318, in function
fn = pfunc(
File "/opt/render/project/src/.venv/lib/python3.10/site-packages/pytensor/compile/function/pfunc.py", line 465, in pfunc
return orig_function(
File "/opt/render/project/src/.venv/lib/python3.10/site-packages/pytensor/compile/function/types.py", line 1762, in orig_function
fn = m.create(defaults)
File "/opt/render/project/src/.venv/lib/python3.10/site-packages/pytensor/compile/function/types.py", line 1654, in create
_fn, _i, _o = self.linker.make_thunk(
File "/opt/render/project/src/.venv/lib/python3.10/site-packages/pytensor/link/basic.py", line 245, in make_thunk
return self.make_all(
File "/opt/render/project/src/.venv/lib/python3.10/site-packages/pytensor/link/vm.py", line 1282, in make_all
vm = self.make_vm(
File "/opt/render/project/src/.venv/lib/python3.10/site-packages/pytensor/link/vm.py", line 1010, in make_vm
from pytensor.link.c.cvm import CVM
File "/opt/render/project/src/.venv/lib/python3.10/site-packages/pytensor/link/c/cvm.py", line 13, in <module>
from pytensor.link.c.lazylinker_c import CLazyLinker
File "/opt/render/project/src/.venv/lib/python3.10/site-packages/pytensor/link/c/lazylinker_c.py", line 122, in <module>
GCC_compiler.compile_str(dirname, code, location=loc, preargs=args)
File "/opt/render/project/src/.venv/lib/python3.10/site-packages/pytensor/link/c/cmodule.py", line 2654, in compile_str
raise CompileError(
pytensor.link.c.exceptions.CompileError: Compilation failed (return status=1):
/usr/bin/g++ -shared -g -Wno-c++11-narrowing -fno-exceptions -fno-unwind-tables -fno-asynchronous-unwind-tables -march=znver3 -mmmx -mpopcnt -msse -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mavx -mavx2 -msse4a -mno-fma4 -mno-xop -mfma -mno-avx512f -mbmi -mbmi2 -maes -mpclmul -mno-avx512vl -mno-avx512bw -mno-avx512dq -mno-avx512cd -mno-avx512er -mno-avx512pf -mno-avx512vbmi -mno-avx512ifma -mno-avx5124vnniw -mno-avx5124fmaps -mno-avx512vpopcntdq -mno-avx512vbmi2 -mno-gfni -mvpclmulqdq -mno-avx512vnni -mno-avx512bitalg -mno-avx512bf16 -mno-avx512vp2intersect -mno-3dnow -madx -mabm -mno-cldemote -mclflushopt -mclwb -mclzero -mcx16 -mno-enqcmd -mf16c -mfsgsbase -mfxsr -mno-hle -msahf -mno-lwp -mlzcnt -mmovbe -mno-movdir64b -mno-movdiri -mno-mwaitx -mno-pconfig -mno-pku -mno-prefetchwt1 -mprfchw -mno-ptwrite -mrdpid -mrdrnd -mrdseed -mno-rtm -mno-serialize -mno-sgx -msha -mno-shstk -mno-tbm -mno-tsxldtrk -mvaes -mno-waitpkg -mwbnoinvd -mxsave -mxsavec -mxsaveopt -mno-xsaves -mno-amx-tile -mno-amx-int8 -mno-amx-bf16 -mno-uintr -mno-hreset -mno-kl -mno-widekl -mno-avxvnni -mno-avx512fp16 --param l1-cache-size=32 --param l1-cache-line-size=64 --param l2-cache-size=512 -mtune=znver3 -DNPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION -m64 -fPIC -I/opt/render/project/src/.venv/lib/python3.10/site-packages/numpy/core/include -I/opt/render/project/python/Python-3.10.6/include/python3.10 -I/opt/render/project/src/.venv/lib/python3.10/site-packages/pytensor/link/c/c_code -L/opt/render/project/python/Python-3.10.6/lib -fvisibility=hidden -o /opt/render/.pytensor/compiledir_Linux-6.8--aws-x86_64-with-glibc2.36--3.10.6-64/lazylinker_ext/lazylinker_ext.so /opt/render/.pytensor/compiledir_Linux-6.8--aws-x86_64-with-glibc2.36--3.10.6-64/lazylinker_ext/mod.cpp -lpython3.10
/usr/bin/ld: /opt/render/project/python/Python-3.10.6/lib/libpython3.10.a(object.o): warning: relocation against `PyDictKeys_Type' in read-only section `.text'
/usr/bin/ld: /opt/render/project/python/Python-3.10.6/lib/libpython3.10.a(bytearrayobject.o): relocation R_X86_64_PC32 against symbol `_Py_NoneStruct' can not be used when making a shared object; recompile with -fPIC
/usr/bin/ld: final link failed: bad value
collect2: error: ld returned 1 exit status
It looks like there is something wrong with lazylinker although I cannot tell what it is exactly. Unfortunately, I cannot reproduce the problem locally. I tried creating a virtual environment and installing the same dependencies in there but the code runs as expected and the lazylinker versions match.
Here is a list of dependencies that were installed successfully installed on Render:
Successfully installed Brotli-1.1.0 Flask-3.0.3 Flask-Compress-1.15 Jinja2-3.1.4 MarkupSafe-2.1.5 Theano-1.0.5 Werkzeug-3.0.4 arviz-0.19.0 blinker-1.8.2 brotlipy-0.7.0 cachetools-5.5.0 certifi-2024.8.30 cffi-1.17.1 charset-normalizer-3.3.2 click-8.1.7 cloudpickle-3.0.0 cons-0.4.6 contourpy-1.3.0 cycler-0.12.1 dash-2.18.1 dash-core-components-2.0.0 dash-html-components-2.0.0 dash-table-5.0.0 dm-tree-0.1.8 et-xmlfile-1.1.0 etuples-0.3.9 filelock-3.16.0 fonttools-4.53.1 formulae-0.5.4 gunicorn-23.0.0 h5netcdf-1.3.0 h5py-3.11.0 idna-3.10 importlib-metadata-8.5.0 itsdangerous-2.2.0 kiwisolver-1.4.7 logical-unification-0.4.6 markdown-it-py-3.0.0 matplotlib-3.9.2 mdurl-0.1.2 miniKanren-1.0.3 multipledispatch-1.0.0 nest-asyncio-1.6.0 numpy-1.23.1 openpyxl-3.1.5 packaging-24.1 pandas-2.2.2 pillow-10.4.0 plotly-5.24.1 pycosat-0.6.6 pycparser-2.22 pygments-2.18.0 pymc-5.16.2 pyparsing-3.1.4 pytensor-2.25.4 python-dateutil-2.9.0.post0 pytz-2024.2 requests-2.32.3 retrying-1.3.4 rich-13.8.1 scipy-1.13.1 six-1.16.0 tenacity-9.0.0 threadpoolctl-3.5.0 toolz-0.12.1 tqdm-4.66.5 typing-extensions-4.12.2 tzdata-2024.1 urllib3-2.2.3 xarray-2024.7.0 xarray-einstats-0.7.0 zipp-3.20.2 zstandard-0.23.0
I tried PyMC versions 4 and 5 but the problem occurred for both.
Do you have any ideas what might solve this problem?
Cheers,
Michael
PS: Perhaps I should add the bit of code where it actually crashes. It fails on the last line of this:
with pm.Model() as self.model:
betas = pm.Normal("betas", mu=prior_mu, sigma=10)
design_matrix = pm.Data("design_matrix", self.mx_ex)
theta = design_matrix.dot(betas)
y_obs = pm.Data("y_obs", self.mx_en["y"].values,
dims="idx")
## Define Poisson likelihood
goals_hat = pm.Poisson("y_hat", mu=pm.math.exp(theta),
observed=y_obs)
approx = pm.fit(progressbar=False)
self.inference = approx.sample(nsamples)