pyMC v 5.0.1 fails test because of lazylinker

Hello, I just upgraded to pyMC 5.0.1 and looking forward to start using it, but my old tests (that worked with v3 and v4) are failing and I can’t run any inference. Here is the test function i’m running:

def test_pymc_xarray():
    '''pyMC uses xarrays and each parameter can have coordinates and dimensions.
    This snippet is meant to explore the xarray structure used in pyMC with a simple example.
    Two approaches are used to create the xarray obsjects and these are compared in terms of time and performance'''

    # import needed packages
    import pymc as pm
    import numpy as np
    import arviz as az
    plot = True # whether to plot the resulting traces

    # set dimensions and create random state for reproducibility
    dims = 50
    rng = np.random.default_rng(5082022)

    # create arbitrary data
    dip_data = rng.standard_normal(dims)
    azimuth_data = rng.standard_normal(dims)

    # run the inference using dimensions and coordinates
    dimension_names = ['Dim ' + str(d) for d in range(dims)]
    with pm.Model(coords={"dip": dimension_names,
                          "azimuth": dimension_names}) as model:
        # initialize parameter vectors
        dips = pm.Normal("dip vector", mu=0, sigma=1, dims='dip')
        azimuths = pm.Normal("azimuth vector", mu=0, sigma=1, dims='azimuth')
        # observed data
        observed_dip = pm.Normal(
            'dip angle',
            mu=0,
            observed=dip_data,
            dims='dip')
        observed_azimuth = pm.Normal(
            'azimuth angle',
            mu=np.zeros(dims),
            observed=azimuth_data,
            dims='azimuth')
        compute_trace_dims = pm.sample(1000, tune=1000, cores=1, random_seed=5082022)

    # run the inference using shape to define dimensions
    with pm.Model() as model2:
        # parameters
        dips2 = pm.Normal('dips', mu=0, sigma=3.0, shape=dims)
        azimuths2 = pm.Normal('azimuths', mu=0, sigma=3.0, shape=dims)
        # observed data
        observed_dip2 = pm.Normal(
            'dip angle',
            mu=0,
            observed=dip_data
        )
        observed_azimuth2 = pm.Normal(
            'azimuth angle',
            mu=0,
            observed=azimuth_data
        )
        compute_trace_shape = pm.sample(1000,
                                        tune=1000,
                                        cores=1,
                                        random_seed=5082022)

    if plot:
        az.plot_trace(compute_trace_dims)
        az.plot_trace(compute_trace_shape)
    compute_trace_dims.sample_stats.acceptance_rate - compute_trace_shape.sample_stats.acceptance_rate

and here is the error I get:

tests/test_probabilistic_sampling.py:24 (test_pymc_xarray)
import errno
    import logging
    import os
    import sys
    import warnings
    from importlib import reload
    from types import ModuleType
    from typing import Optional
    
    import pytensor
    from pytensor.compile.compilelock import lock_ctx
    from pytensor.configdefaults import config
    from pytensor.link.c.cmodule import GCC_compiler
    
    
    _logger = logging.getLogger(__file__)
    
    force_compile = False
    version = 0.212  # must match constant returned in function get_version()
    lazylinker_ext: Optional[ModuleType] = None
    
    
    def try_import():
        global lazylinker_ext
        sys.path[0:0] = [config.compiledir]
        import lazylinker_ext  # noqa
    
        del sys.path[0]
    
    
    def try_reload():
        sys.path[0:0] = [config.compiledir]
        reload(lazylinker_ext)
        del sys.path[0]
    
    
    try:
        # See gh issue #728 for why these lines are here. Summary: compiledir must
        # be at the beginning of the path to avoid conflicts with any other
        # lazylinker_ext modules that might exist (this step handled in try_import
        # and try_reload). An __init__.py file must be created for the same reason.
        # Note that these lines may seem redundant (they are repeated in
        # compile_str()) but if another lazylinker_ext does exist then it will be
        # imported and compile_str won't get called at all.
        location = os.path.join(config.compiledir, "lazylinker_ext")
        if not os.path.exists(location):
            try:
                # Try to make the location
                os.mkdir(location)
            except OSError as e:
                # If we get an error, verify that the error was # 17, the
                # path already exists, and that it is a directory Note: we
                # can't check if it exists before making it, because we
                # are not holding the lock right now, so we could race
                # another process and get error 17 if we lose the race
                assert e.errno == errno.EEXIST
                assert os.path.isdir(location)
    
        init_file = os.path.join(location, "__init__.py")
        if not os.path.exists(init_file):
            try:
                with open(init_file, "w"):
                    pass
            except OSError as e:
                if os.path.exists(init_file):
                    pass  # has already been created
                else:
                    e.args += (f"{location} exist? {os.path.exists(location)}",)
                    raise
    
        _need_reload = False
        if force_compile:
            raise ImportError()
        else:
            try_import()
            _need_reload = True
            actual_version = getattr(lazylinker_ext, "_version", None)
            if version != actual_version:
>               raise ImportError(
                    "Version check of the existing lazylinker compiled file."
                    f" Looking for version {version}, but found {actual_version}. "
                    f"Extra debug information: force_compile={force_compile}, _need_reload={_need_reload}"
                )
E               ImportError: Version check of the existing lazylinker compiled file. Looking for version 0.212, but found 0.211. Extra debug information: force_compile=False, _need_reload=True

I could not find a way to check or upgrade lazylinker. I looked around and similar errors seemed to have occured in earlier versions years ago, but I could not get a good solution.

Looking forward to hear back from the community. Thanks for the help and happy new year to all.

ps. I’m running Ubuntu 22.04 and python 3.10

Is that the full traceback? If not, could you provide it?

Hi Chris, thank you for the quick response (my delay is because ubuntu login crashed for me yesterday…but is now fixed). You’re right, this is not the entire traceback. Here it is:

est_probabilistic_sampling.py::test_pymc_xarray FAILED                  [100%]
tests/test_probabilistic_sampling.py:24 (test_pymc_xarray)
import errno
    import logging
    import os
    import sys
    import warnings
    from importlib import reload
    from types import ModuleType
    from typing import Optional
    
    import pytensor
    from pytensor.compile.compilelock import lock_ctx
    from pytensor.configdefaults import config
    from pytensor.link.c.cmodule import GCC_compiler
    
    
    _logger = logging.getLogger(__file__)
    
    force_compile = False
    version = 0.212  # must match constant returned in function get_version()
    lazylinker_ext: Optional[ModuleType] = None
    
    
    def try_import():
        global lazylinker_ext
        sys.path[0:0] = [config.compiledir]
        import lazylinker_ext  # noqa
    
        del sys.path[0]
    
    
    def try_reload():
        sys.path[0:0] = [config.compiledir]
        reload(lazylinker_ext)
        del sys.path[0]
    
    
    try:
        # See gh issue #728 for why these lines are here. Summary: compiledir must
        # be at the beginning of the path to avoid conflicts with any other
        # lazylinker_ext modules that might exist (this step handled in try_import
        # and try_reload). An __init__.py file must be created for the same reason.
        # Note that these lines may seem redundant (they are repeated in
        # compile_str()) but if another lazylinker_ext does exist then it will be
        # imported and compile_str won't get called at all.
        location = os.path.join(config.compiledir, "lazylinker_ext")
        if not os.path.exists(location):
            try:
                # Try to make the location
                os.mkdir(location)
            except OSError as e:
                # If we get an error, verify that the error was # 17, the
                # path already exists, and that it is a directory Note: we
                # can't check if it exists before making it, because we
                # are not holding the lock right now, so we could race
                # another process and get error 17 if we lose the race
                assert e.errno == errno.EEXIST
                assert os.path.isdir(location)
    
        init_file = os.path.join(location, "__init__.py")
        if not os.path.exists(init_file):
            try:
                with open(init_file, "w"):
                    pass
            except OSError as e:
                if os.path.exists(init_file):
                    pass  # has already been created
                else:
                    e.args += (f"{location} exist? {os.path.exists(location)}",)
                    raise
    
        _need_reload = False
        if force_compile:
            raise ImportError()
        else:
            try_import()
            _need_reload = True
            actual_version = getattr(lazylinker_ext, "_version", None)
            if version != actual_version:
>               raise ImportError(
                    "Version check of the existing lazylinker compiled file."
                    f" Looking for version {version}, but found {actual_version}. "
                    f"Extra debug information: force_compile={force_compile}, _need_reload={_need_reload}"
                )
E               ImportError: Version check of the existing lazylinker compiled file. Looking for version 0.212, but found 0.211. Extra debug information: force_compile=False, _need_reload=True

../../../miniconda3/envs/bedretto/lib/python3.10/site-packages/pytensor/link/c/lazylinker_c.py:79: ImportError

During handling of the above exception, another exception occurred:

    import errno
    import logging
    import os
    import sys
    import warnings
    from importlib import reload
    from types import ModuleType
    from typing import Optional
    
    import pytensor
    from pytensor.compile.compilelock import lock_ctx
    from pytensor.configdefaults import config
    from pytensor.link.c.cmodule import GCC_compiler
    
    
    _logger = logging.getLogger(__file__)
    
    force_compile = False
    version = 0.212  # must match constant returned in function get_version()
    lazylinker_ext: Optional[ModuleType] = None
    
    
    def try_import():
        global lazylinker_ext
        sys.path[0:0] = [config.compiledir]
        import lazylinker_ext  # noqa
    
        del sys.path[0]
    
    
    def try_reload():
        sys.path[0:0] = [config.compiledir]
        reload(lazylinker_ext)
        del sys.path[0]
    
    
    try:
        # See gh issue #728 for why these lines are here. Summary: compiledir must
        # be at the beginning of the path to avoid conflicts with any other
        # lazylinker_ext modules that might exist (this step handled in try_import
        # and try_reload). An __init__.py file must be created for the same reason.
        # Note that these lines may seem redundant (they are repeated in
        # compile_str()) but if another lazylinker_ext does exist then it will be
        # imported and compile_str won't get called at all.
        location = os.path.join(config.compiledir, "lazylinker_ext")
        if not os.path.exists(location):
            try:
                # Try to make the location
                os.mkdir(location)
            except OSError as e:
                # If we get an error, verify that the error was # 17, the
                # path already exists, and that it is a directory Note: we
                # can't check if it exists before making it, because we
                # are not holding the lock right now, so we could race
                # another process and get error 17 if we lose the race
                assert e.errno == errno.EEXIST
                assert os.path.isdir(location)
    
        init_file = os.path.join(location, "__init__.py")
        if not os.path.exists(init_file):
            try:
                with open(init_file, "w"):
                    pass
            except OSError as e:
                if os.path.exists(init_file):
                    pass  # has already been created
                else:
                    e.args += (f"{location} exist? {os.path.exists(location)}",)
                    raise
    
        _need_reload = False
        if force_compile:
            raise ImportError()
        else:
            try_import()
            _need_reload = True
            actual_version = getattr(lazylinker_ext, "_version", None)
            if version != actual_version:
                raise ImportError(
                    "Version check of the existing lazylinker compiled file."
                    f" Looking for version {version}, but found {actual_version}. "
                    f"Extra debug information: force_compile={force_compile}, _need_reload={_need_reload}"
                )
    except ImportError:
        with lock_ctx():
            # Maybe someone else already finished compiling it while we were
            # waiting for the lock?
            try:
                if force_compile:
                    raise ImportError()
                if _need_reload:
                    # The module was successfully imported earlier: we need to
                    # reload it to check if the version was updated.
                    try_reload()
                else:
                    try_import()
                    _need_reload = True
                actual_version = getattr(lazylinker_ext, "_version", None)
                if version != actual_version:
>                   raise ImportError(
                        "Version check of the existing lazylinker compiled file."
                        f" Looking for version {version}, but found {actual_version}. "
                        f"Extra debug information: force_compile={force_compile}, _need_reload={_need_reload}"
                    )
E                   ImportError: Version check of the existing lazylinker compiled file. Looking for version 0.212, but found 0.211. Extra debug information: force_compile=False, _need_reload=True

../../../miniconda3/envs/bedretto/lib/python3.10/site-packages/pytensor/link/c/lazylinker_c.py:100: ImportError

During handling of the above exception, another exception occurred:

    def test_pymc_xarray():
        '''pyMC uses xarrays and each parameter can have coordinates and dimensions.
        This snippet is meant to explore the xarray structure used in pyMC with a simple example.
        Two approaches are used to create the xarray obsjects and these are compared in terms of time and performance'''
    
        # import needed packages
        import pymc as pm
        import numpy as np
        import arviz as az
        plot = True # whether to plot the resulting traces
    
        # set dimensions and create random state for reproducibility
        dims = 50
        rng = np.random.default_rng(5082022)
    
        # create arbitrary data
        dip_data = rng.standard_normal(dims)
        azimuth_data = rng.standard_normal(dims)
    
        # run the inference using dimensions and coordinates
        dimension_names = ['Dim ' + str(d) for d in range(dims)]
        with pm.Model(coords={"dip": dimension_names,
                              "azimuth": dimension_names}) as model:
            # initialize parameter vectors
            dips = pm.Normal("dip vector", mu=0, sigma=1, dims='dip')
            azimuths = pm.Normal("azimuth vector", mu=0, sigma=1, dims='azimuth')
            # observed data
            observed_dip = pm.Normal(
                'dip angle',
                mu=0,
                observed=dip_data,
                dims='dip')
            observed_azimuth = pm.Normal(
                'azimuth angle',
                mu=np.zeros(dims),
                observed=azimuth_data,
                dims='azimuth')
>           compute_trace_dims = pm.sample(1000, tune=1000, cores=1, random_seed=5082022)

test_probabilistic_sampling.py:62: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../../../miniconda3/envs/bedretto/lib/python3.10/site-packages/pymc/sampling/mcmc.py:447: in sample
    step = assign_step_methods(model, step, methods=pm.STEP_METHODS, step_kwargs=kwargs)
../../../miniconda3/envs/bedretto/lib/python3.10/site-packages/pymc/sampling/mcmc.py:189: in assign_step_methods
    return instantiate_steppers(model, steps, selected_steps, step_kwargs)
../../../miniconda3/envs/bedretto/lib/python3.10/site-packages/pymc/sampling/mcmc.py:107: in instantiate_steppers
    step = step_class(vars=vars, model=model, **args)
../../../miniconda3/envs/bedretto/lib/python3.10/site-packages/pymc/step_methods/hmc/nuts.py:182: in __init__
    super().__init__(vars, **kwargs)
../../../miniconda3/envs/bedretto/lib/python3.10/site-packages/pymc/step_methods/hmc/base_hmc.py:109: in __init__
    super().__init__(vars, blocked=blocked, model=self._model, dtype=dtype, **pytensor_kwargs)
../../../miniconda3/envs/bedretto/lib/python3.10/site-packages/pymc/step_methods/arraystep.py:263: in __init__
    func = model.logp_dlogp_function(vars, dtype=dtype, **pytensor_kwargs)
../../../miniconda3/envs/bedretto/lib/python3.10/site-packages/pymc/model.py:642: in logp_dlogp_function
    ip = self.initial_point(0)
../../../miniconda3/envs/bedretto/lib/python3.10/site-packages/pymc/model.py:1126: in initial_point
    fn = make_initial_point_fn(model=self, return_transformed=True)
../../../miniconda3/envs/bedretto/lib/python3.10/site-packages/pymc/initial_point.py:152: in make_initial_point_fn
    func = compile_pymc(inputs=[], outputs=initial_values, mode=pytensor.compile.mode.FAST_COMPILE)
../../../miniconda3/envs/bedretto/lib/python3.10/site-packages/pymc/pytensorf.py:1121: in compile_pymc
    pytensor_function = pytensor.function(
../../../miniconda3/envs/bedretto/lib/python3.10/site-packages/pytensor/compile/function/__init__.py:315: in function
    fn = pfunc(
../../../miniconda3/envs/bedretto/lib/python3.10/site-packages/pytensor/compile/function/pfunc.py:367: in pfunc
    return orig_function(
../../../miniconda3/envs/bedretto/lib/python3.10/site-packages/pytensor/compile/function/types.py:1766: in orig_function
    fn = m.create(defaults)
../../../miniconda3/envs/bedretto/lib/python3.10/site-packages/pytensor/compile/function/types.py:1659: in create
    _fn, _i, _o = self.linker.make_thunk(
../../../miniconda3/envs/bedretto/lib/python3.10/site-packages/pytensor/link/basic.py:254: in make_thunk
    return self.make_all(
../../../miniconda3/envs/bedretto/lib/python3.10/site-packages/pytensor/link/vm.py:1300: in make_all
    vm = self.make_vm(
../../../miniconda3/envs/bedretto/lib/python3.10/site-packages/pytensor/link/vm.py:1021: in make_vm
    from pytensor.link.c.cvm import CVM
../../../miniconda3/envs/bedretto/lib/python3.10/site-packages/pytensor/link/c/cvm.py:13: in <module>
    from pytensor.link.c.lazylinker_c import CLazyLinker
../../../miniconda3/envs/bedretto/lib/python3.10/site-packages/pytensor/link/c/lazylinker_c.py:143: in <module>
    GCC_compiler.compile_str(dirname, code, location=loc, preargs=args)
../../../miniconda3/envs/bedretto/lib/python3.10/site-packages/pytensor/link/c/cmodule.py:2653: in compile_str
    return dlimport(lib_filename)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

fullpath = '/home/alexis/.pytensor/compiledir_Linux-5.15--generic-x86_64-with-glibc2.35-x86_64-3.10.8-64/lazylinker_ext/lazylinker_ext.so'
suffix = '.so'

    def dlimport(fullpath, suffix=None):
        """
        Dynamically load a .so, .pyd, .dll, or .py file.
    
        Parameters
        ----------
        fullpath : str
            A fully-qualified path do a compiled python module.
        suffix : str
            A suffix to strip from the end of fullpath to get the
            import name.
    
        Returns
        -------
        object
            The dynamically loaded module (from __import__).
    
        """
        if not os.path.isabs(fullpath):
            raise ValueError("`fullpath` must be an absolute path", fullpath)
        if suffix is None:
            suffix = ""
    
            dist_suffix = _get_ext_suffix()
            if dist_suffix is not None and dist_suffix != "":
                if fullpath.endswith(dist_suffix):
                    suffix = dist_suffix
    
            if suffix == "":
                if fullpath.endswith(".so"):
                    suffix = ".so"
                elif fullpath.endswith(".pyd"):
                    suffix = ".pyd"
                elif fullpath.endswith(".dll"):
                    suffix = ".dll"
                elif fullpath.endswith(".py"):
                    suffix = ".py"
    
        rval = None
        if fullpath.endswith(suffix):
            module_name = ".".join(fullpath.split(os.path.sep)[-2:])[: -len(suffix)]
        else:
            raise ValueError("path has wrong suffix", (fullpath, suffix))
        workdir = fullpath[: -len(module_name) - 1 - len(suffix)]
    
        _logger.debug(f"WORKDIR {workdir}")
        _logger.debug(f"module_name {module_name}")
    
        sys.path[0:0] = [workdir]  # insert workdir at beginning (temporarily)
        # Explicitly add gcc dll directory on Python 3.8+ on Windows
        if (sys.platform == "win32") & (hasattr(os, "add_dll_directory")):
            gcc_path = shutil.which("gcc")
            if gcc_path is not None:
                os.add_dll_directory(os.path.dirname(gcc_path))
        global import_time
        try:
            importlib.invalidate_caches()
            t0 = time.perf_counter()
            with warnings.catch_warnings():
                warnings.filterwarnings("ignore", message="numpy.ndarray size changed")
                rval = __import__(module_name, {}, {}, [module_name])
            t1 = time.perf_counter()
            import_time += t1 - t0
            if not rval:
                raise Exception("__import__ failed", fullpath)
        finally:
            del sys.path[0]
    
>       assert fullpath.startswith(rval.__file__)
E       AssertionError

../../../miniconda3/envs/bedretto/lib/python3.10/site-packages/pytensor/link/c/cmodule.py:340: AssertionError




======================== 1 failed, 8 warnings in 4.09s =========================

Process finished with exit code 1

Did you install version 5 into a fresh environment? There is some seemingly realted discussion here which suggests it might be an installation issue. But if you installed 5 in a fresh environment (using the recommended installation procedure), I might suggest opening an issue.

You’re right, it is a package conflict of some sort. Having a fresh install works just fine. Thanks for your help!

1 Like