diff --git a/alchemiscale/compute/service.py b/alchemiscale/compute/service.py index f6dd4220..0ac1cff9 100644 --- a/alchemiscale/compute/service.py +++ b/alchemiscale/compute/service.py @@ -15,7 +15,7 @@ from typing import Union, Optional, List, Dict, Tuple from pathlib import Path from threading import Thread -import tempfile +import shutil import requests @@ -266,21 +266,25 @@ def execute(self, task: ScopedKey) -> ScopedKey: # execute the task; this looks the same whether the ProtocolDAG is a # success or failure - shared_tmp = tempfile.TemporaryDirectory( - prefix=f"{str(protocoldag.key)}__", dir=self.shared_basedir - ) - shared = Path(shared_tmp.name) + + shared = self.shared_basedir / str(protocoldag.key) + shared.mkdir() + scratch = self.scratch_basedir / str(protocoldag.key) + scratch.mkdir() protocoldagresult = execute_DAG( protocoldag, - shared=shared, - scratch_basedir=self.scratch_basedir, + shared_basedir=shared, + scratch_basedir=scratch, keep_scratch=self.keep_scratch, raise_error=False, ) if not self.keep_shared: - shared_tmp.cleanup() + shutil.rmtree(shared) + + if not self.keep_scratch: + shutil.rmtree(scratch) # push the result (or failure) back to the compute API result_sk = self.push_result(task, protocoldagresult) diff --git a/alchemiscale/tests/integration/conftest.py b/alchemiscale/tests/integration/conftest.py index 499058b5..bbed8ce1 100644 --- a/alchemiscale/tests/integration/conftest.py +++ b/alchemiscale/tests/integration/conftest.py @@ -277,13 +277,15 @@ def protocoldagresults(tmpdir_factory, transformation): # execute the task with tmpdir_factory.mktemp("protocol_dag").as_cwd(): - shared = Path("shared").absolute() - shared.mkdir() + shared_basedir = Path("shared").absolute() + shared_basedir.mkdir() scratch_basedir = Path("scratch").absolute() scratch_basedir.mkdir() protocoldagresult = execute_DAG( - protocoldag, shared=shared, scratch_basedir=scratch_basedir + protocoldag, + shared_basedir=shared_basedir, + scratch_basedir=scratch_basedir, ) pdrs.append(protocoldagresult) @@ -320,14 +322,14 @@ def protocoldagresults_failure(tmpdir_factory, transformation_failure): # execute the task with tmpdir_factory.mktemp("protocol_dag").as_cwd(): - shared = Path("shared").absolute() - shared.mkdir() + shared_basedir = Path("shared").absolute() + shared_basedir.mkdir() scratch_basedir = Path("scratch").absolute() scratch_basedir.mkdir() protocoldagresult = execute_DAG( protocoldag, - shared=shared, + shared_basedir=shared_basedir, scratch_basedir=scratch_basedir, raise_error=False, ) diff --git a/alchemiscale/tests/integration/interface/client/test_client.py b/alchemiscale/tests/integration/interface/client/test_client.py index 32b5277d..73a7814b 100644 --- a/alchemiscale/tests/integration/interface/client/test_client.py +++ b/alchemiscale/tests/integration/interface/client/test_client.py @@ -252,8 +252,8 @@ def test_cancel_tasks( @staticmethod def _execute_tasks(tasks, n4js, s3os_server): - shared = Path("shared").absolute() - shared.mkdir() + shared_basedir = Path("shared").absolute() + shared_basedir.mkdir() scratch_basedir = Path("scratch").absolute() scratch_basedir.mkdir() @@ -273,10 +273,16 @@ def _execute_tasks(tasks, n4js, s3os_server): name=str(task_sk), ) + shared = shared_basedir / str(protocoldag.key) + shared.mkdir() + + scratch = scratch_basedir / str(protocoldag.key) + scratch.mkdir() + protocoldagresult = execute_DAG( protocoldag, - shared=shared, - scratch_basedir=scratch_basedir, + shared_basedir=shared, + scratch_basedir=scratch, raise_error=False, ) diff --git a/alchemiscale/tests/integration/storage/test_objectstore.py b/alchemiscale/tests/integration/storage/test_objectstore.py index 666e6917..1aca4edf 100644 --- a/alchemiscale/tests/integration/storage/test_objectstore.py +++ b/alchemiscale/tests/integration/storage/test_objectstore.py @@ -2,7 +2,6 @@ import os import pytest -from gufe.protocols.protocoldag import execute_DAG from alchemiscale.models import ScopedKey from alchemiscale.storage import S3ObjectStore diff --git a/alchemiscale/tests/integration/storage/test_statestore.py b/alchemiscale/tests/integration/storage/test_statestore.py index bdc47968..c7e50818 100644 --- a/alchemiscale/tests/integration/storage/test_statestore.py +++ b/alchemiscale/tests/integration/storage/test_statestore.py @@ -1029,13 +1029,15 @@ def test_set_task_result(self, n4js: Neo4jStore, network_tyk2, scope_test, tmpdi # execute the task with tmpdir.as_cwd(): - shared = Path("shared").absolute() - shared.mkdir() + shared_basedir = Path("shared").absolute() + shared_basedir.mkdir() scratch_basedir = Path("scratch").absolute() scratch_basedir.mkdir() protocoldagresult = execute_DAG( - protocoldag, shared=shared, scratch_basedir=scratch_basedir + protocoldag, + shared_basedir=shared_basedir, + scratch_basedir=scratch_basedir, ) pdr_ref = ProtocolDAGResultRef( diff --git a/devtools/conda-envs/docker.yml b/devtools/conda-envs/docker.yml index 98de551e..17e3a73e 100644 --- a/devtools/conda-envs/docker.yml +++ b/devtools/conda-envs/docker.yml @@ -7,13 +7,14 @@ dependencies: - python =3.9 # gufe dependencies - - numpy + - numpy<1.24 # https://github.com/numba/numba/issues/8615#issuecomment-1360792615 - networkx - rdkit - pip - pydantic - openff-toolkit - - openff-units + - openff-units >=0.2.0 + - openff-models >=0.0.4 - openeye-toolkits - typing-extensions @@ -65,9 +66,9 @@ dependencies: - pip: #- git+https://github.com/dotsdl/grolt@relax-cryptography # neo4j test server deployment - - git+https://github.com/OpenFreeEnergy/gufe@alchemiscale-compute + - git+https://github.com/OpenFreeEnergy/gufe - git+https://github.com/OpenFreeEnergy/openfe - git+https://github.com/OpenFreeEnergy/openfe-benchmarks - - git+https://github.com/mikemhenry/openff-models.git@support_nested_models + #- git+https://github.com/mikemhenry/openff-models.git@support_nested_models - git+https://github.com/choderalab/perses.git@protocol-neqcyc #- git+https://github.com/openforcefield/protein-ligand-benchmark diff --git a/devtools/conda-envs/docs.yml b/devtools/conda-envs/docs.yml index 8a94a2dc..899a435b 100644 --- a/devtools/conda-envs/docs.yml +++ b/devtools/conda-envs/docs.yml @@ -7,13 +7,14 @@ dependencies: - python =3.9 # gufe dependencies - - numpy + - numpy<1.24 # https://github.com/numba/numba/issues/8615#issuecomment-1360792615 - networkx - rdkit - pip - pydantic - openff-toolkit - - openff-units + - openff-units >=0.2.0 + - openff-models >=0.0.4 - openeye-toolkits - typing-extensions @@ -63,5 +64,5 @@ dependencies: - git+https://github.com/OpenFreeEnergy/gufe - git+https://github.com/OpenFreeEnergy/openfe - git+https://github.com/OpenFreeEnergy/openfe-benchmarks - - git+https://github.com/mikemhenry/openff-models.git@support_nested_models + #- git+https://github.com/mikemhenry/openff-models.git@support_nested_models #- git+https://github.com/openforcefield/protein-ligand-benchmark diff --git a/devtools/conda-envs/test.yml b/devtools/conda-envs/test.yml index c950268f..88209ff3 100644 --- a/devtools/conda-envs/test.yml +++ b/devtools/conda-envs/test.yml @@ -7,13 +7,14 @@ dependencies: - python =3.9 # gufe dependencies - - numpy + - numpy<1.24 # https://github.com/numba/numba/issues/8615#issuecomment-1360792615 - networkx - rdkit - pip - pydantic - openff-toolkit - - openff-units + - openff-units >=0.2.0 + - openff-models >=0.0.4 - openeye-toolkits - typing-extensions @@ -56,8 +57,8 @@ dependencies: - pip: - git+https://github.com/dotsdl/grolt@relax-cryptography # neo4j test server deployment - - git+https://github.com/OpenFreeEnergy/gufe@alchemiscale-compute + - git+https://github.com/OpenFreeEnergy/gufe - git+https://github.com/OpenFreeEnergy/openfe - - git+https://github.com/dotsdl/openfe-benchmarks@ligandnetwork - - git+https://github.com/mikemhenry/openff-models.git@support_nested_models + - git+https://github.com/OpenFreeEnergy/openfe-benchmarks + #- git+https://github.com/mikemhenry/openff-models.git@support_nested_models #- git+https://github.com/openforcefield/protein-ligand-benchmark