<flow InferenceFlow step slide_point_geometry[0] (input: <s3://mike-test-slide>...)> failed:
    Internal error
Traceback (most recent call last):
  File "/root/micromamba/envs/metaflow_cf9f3ba0982ee4b13002093a7ad13beca8fbc2b6_6898fdd74b61fb799616abbb003f559709fcf2b1/lib/python3.11/site-packages/torch/__init__.py", line 174, in _load_global_deps
    ctypes.CDLL(lib_path, mode=ctypes.RTLD_GLOBAL)
  File "/root/micromamba/envs/metaflow_cf9f3ba0982ee4b13002093a7ad13beca8fbc2b6_6898fdd74b61fb799616abbb003f559709fcf2b1/lib/python3.11/ctypes/__init__.py", line 376, in __init__
    self._handle = _dlopen(self._name, mode)
                   ^^^^^^^^^^^^^^^^^^^^^^^^^
OSError: libcufft.so.11: cannot open shared object file: No such file or directory

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/metaflow/metaflow/cli.py", line 1177, in main
    start(auto_envvar_prefix="METAFLOW", obj=state)
  File "/metaflow/metaflow/tracing/__init__.py", line 27, in wrapper_func
    return func(args, kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/metaflow/metaflow/_vendor/click/core.py", line 829, in __call__
    return self.main(args, kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/metaflow/metaflow/_vendor/click/core.py", line 782, in main
    rv = self.invoke(ctx)
         ^^^^^^^^^^^^^^^^
  File "/metaflow/metaflow/_vendor/click/core.py", line 1259, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/metaflow/metaflow/_vendor/click/core.py", line 1066, in invoke
    return ctx.invoke(self.callback, ctx.params)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/metaflow/metaflow/_vendor/click/core.py", line 610, in invoke
    return callback(args, kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/metaflow/metaflow/_vendor/click/decorators.py", line 21, in new_func
    return f(get_current_context(), args, kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/metaflow/metaflow/cli.py", line 572, in step
    task.run_step(
  File "/metaflow/metaflow/task.py", line 619, in run_step
    self._exec_step_function(step_func)
  File "/metaflow/metaflow/task.py", line 70, in _exec_step_function
    step_function()
  File "/metaflow/infer.py", line 100, in slide_point_geometry
    from eru.models import point_geometries
  File "/metaflow/eru/models/__init__.py", line 1, in <module>
    from .model_config import ModelConfig
  File "/metaflow/eru/models/model_config.py", line 1, in <module>
    from .. import constants
  File "/metaflow/eru/constants.py", line 10, in <module>
    from .utils.collate import default_collate, default_collate_fn_map
  File "/metaflow/eru/utils/collate.py", line 13, in <module>
    import torch
  File "/root/micromamba/envs/metaflow_cf9f3ba0982ee4b13002093a7ad13beca8fbc2b6_6898fdd74b61fb799616abbb003f559709fcf2b1/lib/python3.11/site-packages/torch/__init__.py", line 234, in <module>
    _load_global_deps()
  File "/root/micromamba/envs/metaflow_cf9f3ba0982ee4b13002093a7ad13beca8fbc2b6_6898fdd74b61fb799616abbb003f559709fcf2b1/lib/python3.11/site-packages/torch/__init__.py", line 195, in _load_global_deps
    _preload_cuda_deps(lib_folder, lib_name)
  File "/root/micromamba/envs/metaflow_cf9f3ba0982ee4b13002093a7ad13beca8fbc2b6_6898fdd74b61fb799616abbb003f559709fcf2b1/lib/python3.11/site-packages/torch/__init__.py", line 160, in _preload_cuda_deps
    raise ValueError(f"{lib_name} not found in the system path {sys.path}")
ValueError: libcublas.so.[0-9] not found in the system path ['/metaflow', '/metaflow/_escape_trampolines', '/metaflow', '/root/micromamba/envs/metaflow_cf9f3ba0982ee4b13002093a7ad13beca8fbc2b6_6898fdd74b61fb799616abbb003f559709fcf2b1/lib/python311.zip', '/root/micromamba/envs/metaflow_cf9f3ba0982ee4b13002093a7ad13beca8fbc2b6_6898fdd74b61fb799616abbb003f559709fcf2b1/lib/python3.11', '/root/micromamba/envs/metaflow_cf9f3ba0982ee4b13002093a7ad13beca8fbc2b6_6898fdd74b61fb799616abbb003f559709fcf2b1/lib/python3.11/lib-dynload', '/root/micromamba/envs/metaflow_cf9f3ba0982ee4b13002093a7ad13beca8fbc2b6_6898fdd74b61fb799616abbb003f559709fcf2b1/lib/python3.11/site-packages']

    AWS Batch error:
    Essential container in task exited This could be a transient error. Use @retry to retry.

Task failed.

"export PYTHONPATH=$(pwd)/_escape_trampolines:$(printenv PYTHONPATH)",
                # NOTE: Assumes here that remote nodes are Linux
                "if [[ -n $(printenv LD_LIBRARY_PATH) ]]; then "
                "export MF_ORIG_LD_LIBRARY_PATH=$(printenv LD_LIBRARY_PATH); "
                "export LD_LIBRARY_PATH=$(cat _env_path)/lib:$(printenv LD_LIBRARY_PATH); fi",
                "echo 'Environment bootstrapped.'",

fi

@environment

LD_LIBRARY_PATH

@environment(
        vars={
            "LD_LIBRARY_PATH": "/tmp"
})

@pypi(python=PYTHON_VERSION, packages=PACKAGES, extra_indices=["<https://download.pytorch.org/whl/cu118>"])

"LD_LIBRARY_PATH": "/tmp"

extra_indices=["<https://download.pytorch.org/whl/cu118>"]

@pypi(python=PYTHON_VERSION, packages=PACKAGES, extra_indices=["<https://download.pytorch.org/whl/cu118>"])`

"LD_LIBRARY_PATH": "/tmp",

import torch

if torch.cuda.is_available():
    print("CUDA is available")
else:
    print("CUDA is not available")

"LD_LIBRARY_PATH": "/tmp",

def run_nvidia_smi(query):
            """
            Run the nvidia-smi command with the specified query and return the output.
            """
            cmd = ["nvidia-smi", "--query-gpu={}".format(query), "--format=csv,noheader,nounits"]
            output = subprocess.check_output(cmd).decode("utf-8")
            return output

        # Get GPU names
        gpu_names = run_nvidia_smi("name")
        print("GPU Names:")
        print(gpu_names)

Tesla T4

print(f"Found {device_count} CUDA devices")
        for i in range(device_count):
            device_name = torch.cuda.get_device_name(i)
            print(f"Device {i}: {device_name}")

RuntimeError: Found no NVIDIA driver on your system. Please check that you have an NVIDIA GPU and installed a driver from <http://www.nvidia.com/Download/index.aspx>

@environment(
        vars={
            "LD_LIBRARY_PATH": "/tmp",
            "NVIDIA_DRIVER_CAPABILITIES": "compute,utility",
        }
    )