Environment
I have two servers: ServerA and ServerB.
ServerA has three Alevo U280 and ServerB has one Alevo U280.
On ServerA I run dask-scheduler and dask-worker tcp://192.168.1.246:8786
On ServerB, I run dask-worker tcp://192.168.1.246:8786
The commands above are run in conda environment in terminal.
The conda environment of both servers are the same.
The version of xrt, dask and pynq:
Platform: xilinx_u280_gen3x16_xdma_1_202211_1
Vitis: 2022.2
XRT Version : 2.14.354
dask, version 2023.6.0
PYNQ version: 3.0.1
Version of Ubuntu:
ServerA: 18.04
ServerB: 20.04
All conda environment:
environment.txt
Error
When I run
daskdev_w0 = DaskDevice(client, workers[0])
daskdev_w1 = DaskDevice(client, workers[1])
xclbin = '../benchmark.intf1.xilinx_u280_gen3x16_xdma_1_202211_1/vnx_benchmark_if1.xclbin'
ol_w0 = pynq.Overlay(xclbin, device=daskdev_w0)
ol_w1 = pynq.Overlay(xclbin, device=daskdev_w1)
in vnx-benchmark-throughput.ipynb, I got this error:
---------------------------------------------------------------------------
RuntimeError Traceback (most recent call last)
Cell In[29], line 1
----> 1 ol_w0 = pynq.Overlay(xclbin, device=daskdev_w0)
2 ol_w1 = pynq.Overlay(xclbin, device=daskdev_w1)
File [~/anaconda3/envs/xup/lib/python3.9/site-packages/pynq/overlay.py:348](https://vscode-remote+ssh-002dremote-002bfpga-002depcc.vscode-resource.vscode-cdn.net/home/chenqi/code/xup_vitis_network_example/Notebooks/~/anaconda3/envs/xup/lib/python3.9/site-packages/pynq/overlay.py:348), in Overlay.__init__(self, bitfile_name, dtbo, download, ignore_version, device, gen_cache)
345 self.systemgraph = None
347 if download:
--> 348 self.download()
349 else:
350 if gen_cache:
File [~/anaconda3/envs/xup/lib/python3.9/site-packages/pynq/overlay.py:428](https://vscode-remote+ssh-002dremote-002bfpga-002depcc.vscode-resource.vscode-cdn.net/home/chenqi/code/xup_vitis_network_example/Notebooks/~/anaconda3/envs/xup/lib/python3.9/site-packages/pynq/overlay.py:428), in Overlay.download(self, dtbo)
425 else:
426 Clocks.set_pl_clk(i)
--> 428 super().download(self.parser)
429 if dtbo:
430 super().insert_dtbo(dtbo)
File [~/anaconda3/envs/xup/lib/python3.9/site-packages/pynq/bitstream.py:168](https://vscode-remote+ssh-002dremote-002bfpga-002depcc.vscode-resource.vscode-cdn.net/home/chenqi/code/xup_vitis_network_example/Notebooks/~/anaconda3/envs/xup/lib/python3.9/site-packages/pynq/bitstream.py:168), in Bitstream.download(self, parser)
151 def download(self, parser=None):
152 """Download the bitstream onto PL and update PL information.
153
154 If device tree blob has been specified during initialization, this
(...)
166
167 """
--> 168 self.device.download(self, parser)
File [~/code/xup_vitis_network_example/Notebooks/dask_pynq.py:160](https://vscode-remote+ssh-002dremote-002bfpga-002depcc.vscode-resource.vscode-cdn.net/home/chenqi/code/xup_vitis_network_example/Notebooks/~/code/xup_vitis_network_example/Notebooks/dask_pynq.py:160), in DaskDevice.download(self, bitstream, parser)
158 with open(bitstream.bitfile_name, 'rb') as f:
159 bitstream_data = f.read()
--> 160 self._call_dask(_download, bitstream_data)
161 super().post_download(bitstream, parser)
File [~/code/xup_vitis_network_example/Notebooks/dask_pynq.py:124](https://vscode-remote+ssh-002dremote-002bfpga-002depcc.vscode-resource.vscode-cdn.net/home/chenqi/code/xup_vitis_network_example/Notebooks/~/code/xup_vitis_network_example/Notebooks/dask_pynq.py:124), in DaskDevice._call_dask(self, func, *args)
121 def _call_dask(self, func, *args):
122 future = self._dask_client.submit(func, *args, workers=self._worker,
123 pure=False)
--> 124 return future.result()
File [~/anaconda3/envs/xup/lib/python3.9/site-packages/distributed/client.py:326](https://vscode-remote+ssh-002dremote-002bfpga-002depcc.vscode-resource.vscode-cdn.net/home/chenqi/code/xup_vitis_network_example/Notebooks/~/anaconda3/envs/xup/lib/python3.9/site-packages/distributed/client.py:326), in Future.result(self, timeout)
324 if self.status == "error":
325 typ, exc, tb = result
--> 326 raise exc.with_traceback(tb)
327 elif self.status == "cancelled":
328 raise result
File [~/code/xup_vitis_network_example/Notebooks/dask_pynq.py:73](https://vscode-remote+ssh-002dremote-002bfpga-002depcc.vscode-resource.vscode-cdn.net/home/chenqi/code/xup_vitis_network_example/Notebooks/~/code/xup_vitis_network_example/Notebooks/dask_pynq.py:73), in _download()
71 f.write(bitstream_data)
72 f.flush()
---> 73 ol = pynq.Overlay(f.name)
File [~/anaconda3/envs/xup/lib/python3.9/site-packages/pynq/overlay.py:315](https://vscode-remote+ssh-002dremote-002bfpga-002depcc.vscode-resource.vscode-cdn.net/home/chenqi/code/xup_vitis_network_example/Notebooks/~/anaconda3/envs/xup/lib/python3.9/site-packages/pynq/overlay.py:315), in __init__()
286 def __init__(
287 self, bitfile_name, dtbo=None, download=True, ignore_version=False, device=None, gen_cache=False
288 ):
289 """Return a new Overlay object.
290
291 An overlay instantiates a bitstream object as a member initially.
(...)
313
314 """
--> 315 super().__init__(bitfile_name, dtbo, partial=False, device=device)
317 self._register_drivers()
319 self.device.set_bitfile_name(self.bitfile_name)
File [~/anaconda3/envs/xup/lib/python3.9/site-packages/pynq/bitstream.py:88](https://vscode-remote+ssh-002dremote-002bfpga-002depcc.vscode-resource.vscode-cdn.net/home/chenqi/code/xup_vitis_network_example/Notebooks/~/anaconda3/envs/xup/lib/python3.9/site-packages/pynq/bitstream.py:88), in __init__()
85 if device is None:
86 from .pl_server.device import Device
---> 88 device = Device.active_device
89 self.device = device
91 # self.xsa = None
92 # if bitfile_name.endswith(".xsa"):
93 # self.xsa_filepath = bitfile_name
94 # self.xsa = pynqutils.build_utils.XsaParser(bitfile_name)
95 # bitfile_name = self.xsa.bitstreamPaths[0]
96 # self.xsa.load_bdc_metadata()
File [~/anaconda3/envs/xup/lib/python3.9/site-packages/pynq/pl_server/device.py:70](https://vscode-remote+ssh-002dremote-002bfpga-002depcc.vscode-resource.vscode-cdn.net/home/chenqi/code/xup_vitis_network_example/Notebooks/~/anaconda3/envs/xup/lib/python3.9/site-packages/pynq/pl_server/device.py:70), in active_device()
63 """The device used by PYNQ if `None` used for a device parameter
64
65 This defaults to the device with the lowest priority and index but
66 can be overridden to globally change the default.
67
68 """
69 if not hasattr(cls, "_active_device"):
---> 70 if len(cls.devices) == 0:
71 raise RuntimeError("No Devices Found")
72 cls._active_device = cls.devices[0]
File [~/anaconda3/envs/xup/lib/python3.9/site-packages/pynq/pl_server/device.py:54](https://vscode-remote+ssh-002dremote-002bfpga-002depcc.vscode-resource.vscode-cdn.net/home/chenqi/code/xup_vitis_network_example/Notebooks/~/anaconda3/envs/xup/lib/python3.9/site-packages/pynq/pl_server/device.py:54), in devices()
52 cls._devices = []
53 for key in sorted(DeviceMeta._subclasses.keys()):
---> 54 cls._devices.extend(DeviceMeta._subclasses[key]._probe_())
55 if len(cls._devices) == 0 and "XILINX_XRT" not in os.environ:
56 warnings.warn(
57 "No devices found, is the XRT environment sourced?", UserWarning
58 )
File [~/anaconda3/envs/xup/lib/python3.9/site-packages/pynq/pl_server/xrt_device.py:304](https://vscode-remote+ssh-002dremote-002bfpga-002depcc.vscode-resource.vscode-cdn.net/home/chenqi/code/xup_vitis_network_example/Notebooks/~/anaconda3/envs/xup/lib/python3.9/site-packages/pynq/pl_server/xrt_device.py:304), in _probe_()
302 return []
303 num = xrt.xclProbe()
--> 304 devices = [XrtDevice(i) for i in range(num)]
305 return devices
File [~/anaconda3/envs/xup/lib/python3.9/site-packages/pynq/pl_server/xrt_device.py:304](https://vscode-remote+ssh-002dremote-002bfpga-002depcc.vscode-resource.vscode-cdn.net/home/chenqi/code/xup_vitis_network_example/Notebooks/~/anaconda3/envs/xup/lib/python3.9/site-packages/pynq/pl_server/xrt_device.py:304), in ()
302 return []
303 num = xrt.xclProbe()
--> 304 devices = [XrtDevice(i) for i in range(num)]
305 return devices
File [~/anaconda3/envs/xup/lib/python3.9/site-packages/pynq/pl_server/xrt_device.py:325](https://vscode-remote+ssh-002dremote-002bfpga-002depcc.vscode-resource.vscode-cdn.net/home/chenqi/code/xup_vitis_network_example/Notebooks/~/anaconda3/envs/xup/lib/python3.9/site-packages/pynq/pl_server/xrt_device.py:325), in __init__()
323 self.active_bos = []
324 self._bo_cache = []
--> 325 self._loop = asyncio.get_event_loop()
326 self._streams = {}
File [~/anaconda3/envs/xup/lib/python3.9/asyncio/events.py:642](https://vscode-remote+ssh-002dremote-002bfpga-002depcc.vscode-resource.vscode-cdn.net/home/chenqi/code/xup_vitis_network_example/Notebooks/~/anaconda3/envs/xup/lib/python3.9/asyncio/events.py:642), in get_event_loop()
639 self.set_event_loop(self.new_event_loop())
641 if self._local._loop is None:
--> 642 raise RuntimeError('There is no current event loop in thread %r.'
643 % threading.current_thread().name)
645 return self._local._loop
RuntimeError: There is no current event loop in thread 'Dask-Default-Threads-55251-0'.
In terminal:
