From 93e1ae4696aed9331847f815c2fd68e0047b7661 Mon Sep 17 00:00:00 2001 From: Brian Cherinka Date: Fri, 5 Jan 2024 12:45:11 -0500 Subject: [PATCH 1/2] bug fix for rsync on remote compressed file --- CHANGELOG.rst | 4 ++++ python/sdss_access/sync/rsync.py | 6 +++--- tests/sync/test_rsync.py | 16 ++++++++++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a25efd9..9b32c14 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,10 @@ Change Log This document records the main changes to the sdss_access code. +3.0.4 (unreleased) +------------------ +- Fix issue `52` - rsync failure when remote file is compressed compared to template + 3.0.3 (11-29-2023) ------------------ - Add new ``tilegrp`` method for grouping LVM tile ids diff --git a/python/sdss_access/sync/rsync.py b/python/sdss_access/sync/rsync.py index 8cc2992..ddf0ad8 100644 --- a/python/sdss_access/sync/rsync.py +++ b/python/sdss_access/sync/rsync.py @@ -13,9 +13,9 @@ class RsyncAccess(BaseAccess): remote_scheme = 'rsync' access_mode = 'rsync' - def __init__(self, label='sdss_rsync', stream_count=5, mirror=False, public=False, release=None, + def __init__(self, label='sdss_rsync', stream_count=5, mirror=False, public=False, release=None, verbose=False): - super(RsyncAccess, self).__init__(stream_count=stream_count, mirror=mirror, public=public, + super(RsyncAccess, self).__init__(stream_count=stream_count, mirror=mirror, public=public, release=release, verbose=verbose) self.label = label self.auth = None @@ -29,7 +29,7 @@ def __repr__(self): def get_task_out(self, task=None): if task: - command = "rsync -R %(source)s" % task + command = "rsync -R %(source)s*" % task if self.verbose: print(command) status, out, err = self.stream.cli.foreground_run(command) diff --git a/tests/sync/test_rsync.py b/tests/sync/test_rsync.py index 6bcc8d7..8ec2502 100644 --- a/tests/sync/test_rsync.py +++ b/tests/sync/test_rsync.py @@ -55,6 +55,22 @@ def test_release_from_module(self, monkeypatch, tree_ver, exp): assert rsync.release == tree_ver assert exp in loc + def test_remote_compression(self): + """ test we can find the correct file when the remote file is compressed """ + rsync = RsyncAccess(release='ipl3') + rsync.remote() + + # template path is not compressed + ff = rsync.templates["mwmAllStar"] + assert ff.endswith('summary/mwmAllStar-{v_astra}.fits') + + # remote source is fully resolved path + rsync.add("mwmAllStar", v_astra="0.5.0") + rsync.set_stream() + source = rsync.initial_stream.task[0]['source'] + assert source.endswith('0.5.0/summary/mwmAllStar-0.5.0.fits.gz') + + class TestRsyncFails(object): From 5f02a45cedc44c48d5b104096efd78bfd21d4b3a Mon Sep 17 00:00:00 2001 From: Brian Cherinka Date: Fri, 5 Jan 2024 12:51:38 -0500 Subject: [PATCH 2/2] fixing test --- tests/sync/test_rsync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/sync/test_rsync.py b/tests/sync/test_rsync.py index 8ec2502..7b9e712 100644 --- a/tests/sync/test_rsync.py +++ b/tests/sync/test_rsync.py @@ -67,7 +67,7 @@ def test_remote_compression(self): # remote source is fully resolved path rsync.add("mwmAllStar", v_astra="0.5.0") rsync.set_stream() - source = rsync.initial_stream.task[0]['source'] + source = rsync.stream.task[0]['source'] assert source.endswith('0.5.0/summary/mwmAllStar-0.5.0.fits.gz')