Moving that directory has changed something. Now, according to qstat the cluster is running:
150687455.gadi-pbs dr6273 normal-* dask-work* 19772* 1 1 4096m 00:05 R 00:03
But in jupyter there is nothing:
sys.path has a new line:
['/jobfs/150687205.gadi-pbs/dask-scratch-space/scheduler-eszpym4_',
'/home/599/dr6273',
'/g/data/w42/dr6273/work/xbootstrap',
'/g/data/w42/dr6273/work/xeof',
'/home/599/dr6273/ondemand/data/sys/dashboard/batch_connect/sys/jupyter/ncigadi/output/adfc5c51-185b-4195-9f3d-8766238bd90a/lib/python3',
'/g/data/xp65/public/apps/med_conda/envs/analysis3-25.08/lib/python311.zip',
'/g/data/xp65/public/apps/med_conda/envs/analysis3-25.08/lib/python3.11',
'/g/data/xp65/public/apps/med_conda/envs/analysis3-25.08/lib/python3.11/lib-dynload',
'',
'/g/data/xp65/public/apps/med_conda/envs/analysis3-25.08/lib/python3.11/site-packages']
Here is the output of dask.config.config
{'jobqueue': {'pbs': {'local-directory': '$TMPDIR', 'log-directory': '/g/data/w42/dr6273/tmp/logs', 'python': '/g/data/xp65/public/apps/med_conda_scripts/analysis3-25.08.d/bin/python', 'name': 'dask-worker', 'cores': None, 'memory': None, 'processes': None, 'interface': None, 'death-timeout': 60, 'shared-temp-directory': None, 'extra': None, 'worker-command': 'distributed.cli.dask_worker', 'worker-extra-args': [], 'shebang': '#!/usr/bin/env bash', 'queue': None, 'account': None, 'walltime': '00:30:00', 'env-extra': None, 'job-script-prologue': [], 'resource-spec': None, 'job-extra': None, 'job-extra-directives': [], 'job-directives-skip': [], 'scheduler-options': {}}, 'oar': {'name': 'dask-worker', 'cores': None, 'memory': None, 'processes': None, 'python': None, 'interface': None, 'death-timeout': 60, 'local-directory': None, 'shared-temp-directory': None, 'extra': None, 'worker-command': 'distributed.cli.dask_worker', 'worker-extra-args': [], 'shebang': '#!/usr/bin/env bash', 'queue': None, 'project': None, 'walltime': '00:30:00', 'env-extra': None, 'job-script-prologue': [], 'resource-spec': None, 'job-extra': None, 'job-extra-directives': [], 'job-directives-skip': [], 'log-directory': None, 'memory-per-core-property-name': None, 'scheduler-options': {}}, 'sge': {'name': 'dask-worker', 'cores': None, 'memory': None, 'processes': None, 'python': None, 'interface': None, 'death-timeout': 60, 'local-directory': None, 'shared-temp-directory': None, 'extra': None, 'worker-command': 'distributed.cli.dask_worker', 'worker-extra-args': [], 'shebang': '#!/usr/bin/env bash', 'queue': None, 'project': None, 'walltime': '00:30:00', 'env-extra': None, 'job-script-prologue': [], 'job-extra': None, 'job-extra-directives': [], 'job-directives-skip': [], 'log-directory': None, 'resource-spec': None, 'scheduler-options': {}}, 'slurm': {'name': 'dask-worker', 'cores': None, 'memory': None, 'processes': None, 'python': None, 'interface': None, 'death-timeout': 60, 'local-directory': None, 'shared-temp-directory': None, 'extra': None, 'worker-command': 'distributed.cli.dask_worker', 'worker-extra-args': [], 'shebang': '#!/usr/bin/env bash', 'queue': None, 'account': None, 'walltime': '00:30:00', 'env-extra': None, 'job-script-prologue': [], 'job-cpu': None, 'job-mem': None, 'job-extra': None, 'job-extra-directives': [], 'job-directives-skip': [], 'log-directory': None, 'scheduler-options': {}}, 'moab': {'name': 'dask-worker', 'cores': None, 'memory': None, 'processes': None, 'python': None, 'interface': None, 'death-timeout': 60, 'local-directory': None, 'shared-temp-directory': None, 'extra': None, 'worker-command': 'distributed.cli.dask_worker', 'worker-extra-args': [], 'shebang': '#!/usr/bin/env bash', 'queue': None, 'account': None, 'walltime': '00:30:00', 'env-extra': None, 'job-script-prologue': [], 'resource-spec': None, 'job-extra': None, 'job-extra-directives': [], 'job-directives-skip': [], 'log-directory': None, 'scheduler-options': {}}, 'lsf': {'name': 'dask-worker', 'cores': None, 'memory': None, 'processes': None, 'python': None, 'interface': None, 'death-timeout': 60, 'local-directory': None, 'shared-temp-directory': None, 'extra': None, 'worker-command': 'distributed.cli.dask_worker', 'worker-extra-args': [], 'shebang': '#!/usr/bin/env bash', 'queue': None, 'project': None, 'walltime': '00:30', 'env-extra': None, 'job-script-prologue': [], 'ncpus': None, 'mem': None, 'job-extra': None, 'job-extra-directives': [], 'job-directives-skip': [], 'log-directory': None, 'lsf-units': None, 'use-stdin': True, 'scheduler-options': {}}, 'htcondor': {'name': 'dask-worker', 'cores': None, 'memory': None, 'processes': None, 'python': None, 'interface': None, 'death-timeout': 60, 'local-directory': None, 'shared-temp-directory': None, 'extra': None, 'worker-command': 'distributed.cli.dask_worker', 'worker-extra-args': [], 'disk': None, 'env-extra': None, 'job-script-prologue': [], 'job-extra': None, 'job-extra-directives': {}, 'job-directives-skip': [], 'submit-command-extra': [], 'cancel-command-extra': [], 'log-directory': None, 'shebang': '#!/usr/bin/env condor_submit', 'scheduler-options': {}}, 'local': {'name': 'dask-worker', 'cores': None, 'memory': None, 'processes': None, 'python': None, 'interface': None, 'death-timeout': 60, 'local-directory': None, 'shared-temp-directory': None, 'extra': None, 'worker-command': 'distributed.cli.dask_worker', 'worker-extra-args': [], 'env-extra': None, 'job-script-prologue': [], 'job-extra': None, 'job-extra-directives': [], 'job-directives-skip': [], 'log-directory': None, 'scheduler-options': {}}}, 'distributed': {'dashboard': {'link': '/proxy/{port}/status', 'export-tool': False, 'graph-max-items': 5000, 'prometheus': {'namespace': 'dask'}}, 'version': 2, 'scheduler': {'allowed-failures': 3, 'bandwidth': 100000000, 'blocked-handlers': [], 'contact-address': None, 'default-data-size': '1kiB', 'events-cleanup-delay': '1h', 'idle-timeout': None, 'no-workers-timeout': None, 'work-stealing': True, 'work-stealing-interval': '1s', 'worker-saturation': 1.1, 'rootish-taskgroup': 5, 'rootish-taskgroup-dependencies': 5, 'worker-ttl': '5 minutes', 'preload': [], 'preload-argv': [], 'unknown-task-duration': '500ms', 'default-task-durations': {'rechunk-split': '1us', 'split-shuffle': '1us', 'split-taskshuffle': '1us', 'split-stage': '1us'}, 'validate': False, 'dashboard': {'status': {'task-stream-length': 1000}, 'tasks': {'task-stream-length': 100000}, 'tls': {'ca-file': None, 'key': None, 'cert': None}, 'bokeh-application': {'allow_websocket_origin': ['*'], 'keep_alive_milliseconds': 500, 'check_unused_sessions_milliseconds': 500}}, 'locks': {'lease-validation-interval': '10s', 'lease-timeout': '30s'}, 'http': {'routes': ['distributed.http.scheduler.prometheus', 'distributed.http.scheduler.info', 'distributed.http.scheduler.json', 'distributed.http.health', 'distributed.http.proxy', 'distributed.http.statics']}, 'allowed-imports': ['dask', 'distributed'], 'active-memory-manager': {'start': True, 'interval': '2s', 'measure': 'optimistic', 'policies': [{'class': 'distributed.active_memory_manager.ReduceReplicas'}]}}, 'worker': {'blocked-handlers': [], 'multiprocessing-method': 'spawn', 'use-file-locking': True, 'transfer': {'message-bytes-limit': '50MB'}, 'connections': {'outgoing': 50, 'incoming': 10}, 'preload': [], 'preload-argv': [], 'daemon': True, 'validate': False, 'resources': {}, 'lifetime': {'duration': None, 'stagger': '0 seconds', 'restart': False}, 'profile': {'enabled': True, 'interval': '10ms', 'cycle': '1000ms', 'low-level': False}, 'memory': {'recent-to-old-time': '30s', 'rebalance': {'measure': 'optimistic', 'sender-min': 0.3, 'recipient-max': 0.6, 'sender-recipient-gap': 0.1}, 'transfer': 0.1, 'target': 0.6, 'spill': 0.7, 'pause': 0.8, 'terminate': 0.95, 'max-spill': False, 'spill-compression': 'auto', 'monitor-interval': '100ms'}, 'http': {'routes': ['distributed.http.worker.prometheus', 'distributed.http.health', 'distributed.http.statics']}}, 'nanny': {'preload': [], 'preload-argv': [], 'environ': {}, 'pre-spawn-environ': {'MALLOC_TRIM_THRESHOLD_': 65536, 'OMP_NUM_THREADS': 1, 'MKL_NUM_THREADS': 1, 'OPENBLAS_NUM_THREADS': 1}}, 'client': {'heartbeat': '5s', 'scheduler-info-interval': '2s', 'security-loader': None, 'preload': [], 'preload-argv': []}, 'deploy': {'lost-worker-timeout': '15s', 'cluster-repr-interval': '500ms'}, 'adaptive': {'interval': '1s', 'target-duration': '5s', 'minimum': 0, 'maximum': inf, 'wait-count': 3}, 'comm': {'retry': {'count': 0, 'delay': {'min': '1s', 'max': '20s'}}, 'compression': False, 'shard': '64MiB', 'offload': '10MiB', 'default-scheme': 'tcp', 'socket-backlog': 2048, 'ucx': {'cuda-copy': None, 'tcp': None, 'nvlink': None, 'infiniband': None, 'rdmacm': None, 'create-cuda-context': None, 'environment': {}}, 'zstd': {'level': 3, 'threads': 0}, 'timeouts': {'connect': '30s', 'tcp': '30s'}, 'require-encryption': None, 'tls': {'ciphers': None, 'min-version': 1.2, 'max-version': None, 'ca-file': None, 'scheduler': {'cert': None, 'key': None}, 'worker': {'key': None, 'cert': None}, 'client': {'key': None, 'cert': None}}, 'websockets': {'shard': '8MiB'}}, 'diagnostics': {'nvml': True, 'cudf': False, 'computations': {'max-history': 100, 'nframes': 0, 'ignore-modules': ['asyncio', 'functools', 'threading', 'datashader', 'dask', 'debugpy', 'distributed', 'ipykernel', 'coiled', 'cudf', 'cuml', 'matplotlib', 'pluggy', 'prefect', 'rechunker', 'xarray', 'xgboost', 'xdist', '__channelexec__', 'execnet'], 'ignore-files': ['runpy\\.py', 'pytest', 'py\\.test', 'pytest-script\\.py', '_pytest', 'pycharm', 'vscode_pytest', 'get_output_via_markers\\.py']}, 'erred-tasks': {'max-history': 100}}, 'p2p': {'comm': {'buffer': '1 GiB', 'concurrency': 10, 'message-bytes-limit': '2 MiB', 'retry': {'count': 10, 'delay': {'min': '1s', 'max': '30s'}}}, 'storage': {'buffer': '100 MiB', 'disk': True}, 'threads': None}, 'admin': {'large-graph-warning-threshold': '10MB', 'tick': {'interval': '20ms', 'limit': '3s', 'cycle': '1s'}, 'max-error-length': 10000, 'log-length': 10000, 'log-format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s', 'low-level-log-length': 1000, 'pdb-on-err': False, 'system-monitor': {'interval': '500ms', 'log-length': 7200, 'disk': True, 'host-cpu': False, 'gil': {'enabled': True, 'interval': '1ms'}}, 'event-loop': 'tornado'}, 'rmm': {'pool-size': None}}, 'temporary_directory': '/jobfs/150687205.gadi-pbs', 'visualization': {'engine': None}, 'tokenize': {'ensure-deterministic': False}, 'dataframe': {'backend': 'pandas', 'shuffle': {'method': None, 'compression': None}, 'parquet': {'metadata-task-size-local': 512, 'metadata-task-size-remote': 1, 'minimum-partition-size': 75000000}, 'convert-string': None, 'query-planning': None}, 'array': {'backend': 'numpy', 'chunk-size': '128MiB', 'chunk-size-tolerance': 1.25, 'rechunk': {'method': None, 'threshold': 4}, 'svg': {'size': 120}, 'slicing': {'split-large-chunks': None}, 'query-planning': None}, 'optimization': {'annotations': {'fuse': True}, 'fuse': {'active': None, 'ave-width': 1, 'max-width': None, 'max-height': inf, 'max-depth-new-edges': None, 'rename-keys': True, 'delayed': False}}, 'admin': {'async-client-fallback': None, 'traceback': {'shorten': ['concurrent[\\\\\\/]futures[\\\\\\/]', 'dask[\\\\\\/](base|core|local|multiprocessing|optimization|threaded|utils)\\.py', 'dask[\\\\\\/]array[\\\\\\/]core\\.py', 'dask[\\\\\\/]dataframe[\\\\\\/](core|methods)\\.py', 'dask[\\\\\\/]_task_spec\\.py', 'distributed[\\\\\\/](client|scheduler|utils|worker)\\.py', 'tornado[\\\\\\/]gen\\.py', 'pandas[\\\\\\/]core[\\\\\\/]']}}, 'scheduler': 'dask.distributed'}
