diff --git a/S1_NRB/ard.py b/S1_NRB/ard.py
index 9f7c801..6db92af 100644
--- a/S1_NRB/ard.py
+++ b/S1_NRB/ard.py
@@ -1,1108 +1,1108 @@
-import os
-import re
-import time
-import shutil
-import logging
-import tempfile
-from datetime import datetime, timezone
-import numpy as np
-from lxml import etree
-from time import gmtime, strftime
-from copy import deepcopy
-from scipy.interpolate import griddata
-from osgeo import gdal
-from spatialist import Raster, Vector, vectorize, boundary, bbox, intersect, rasterize
-from spatialist.auxil import gdalwarp, gdalbuildvrt
-from spatialist.ancillary import finder
-from pyroSAR import identify, identify_many
-import S1_NRB
-from S1_NRB import dem, ocn
-from S1_NRB.metadata import extract, xml, stac
-from S1_NRB.metadata.mapping import LERC_ERR_THRES
-from S1_NRB.ancillary import generate_unique_id, vrt_add_overviews
-from S1_NRB.metadata.extract import copy_src_meta, get_src_meta, find_in_annotation
-from S1_NRB.snap import find_datasets
-
-
-def format(config, product_type, scenes, datadir, outdir, tile, extent, epsg, wbm=None, dem_type=None, multithread=True,
- compress=None, overviews=None, kml=None, annotation=None, update=False):
- """
- Finalizes the generation of Sentinel-1 Analysis Ready Data (ARD) products after SAR processing has finished.
- This includes the following:
-
- - Creating all measurement and annotation datasets in Cloud Optimized GeoTIFF (COG) format
- - Creating additional annotation datasets in Virtual Raster Tile (VRT) format
- - Applying the ARD product directory structure & naming convention
- - Generating metadata in XML and JSON formats for the ARD product as well as source SLC datasets
-
- Parameters
- ----------
- config: dict
- Dictionary of the parsed config parameters for the current process.
- product_type: str
- The type of ARD product to be generated. Options: 'NRB' or 'ORB'.
- scenes: list[str]
- List of scenes to process. Either a single scene or multiple, matching scenes (consecutive acquisitions).
- All scenes are expected to overlap with `extent` and an error will be thrown if the processing output
- cannot be found for any of the scenes.
- datadir: str
- The directory containing the SAR datasets processed from the source scenes using pyroSAR.
- outdir: str
- The directory to write the final files to.
- tile: str
- ID of an MGRS tile.
- extent: dict
- Spatial extent of the MGRS tile, derived from a :class:`~spatialist.vector.Vector` object.
- epsg: int
- The CRS used for the ARD product; provided as an EPSG code.
- wbm: str or None
- Path to a water body mask file with the dimensions of an MGRS tile.
- dem_type: str or None
- if defined, a DEM layer will be added to the product. The suffix `em` (elevation model) is used.
- Default `None`: do not add a DEM layer.
- multithread: bool
- Should `gdalwarp` use multithreading? Default is True. The number of threads used, can be adjusted in the
- `config.ini` file with the parameter `gdal_threads`.
- compress: str or None
- Compression algorithm to use. See https://gdal.org/drivers/raster/gtiff.html#creation-options for options.
- Defaults to 'LERC_DEFLATE'.
- overviews: list[int] or None
- Internal overview levels to be created for each GeoTIFF file. Defaults to [2, 4, 9, 18, 36]
- kml: str or None
- The KML file containing the MGRS tile geometries. Only needs to be defined if `dem_type!=None`.
- annotation: list[str] or None
- an optional list to select the annotation layers. Default `None`: create all layers if the
- source products contain the required input layers. Options:
-
- - dm: data mask (four masks: not layover not shadow, layover, shadow, ocean water)
- - ei: ellipsoidal incident angle
- - em: digital elevation model
- - id: acquisition ID image (source scene ID per pixel)
- - lc: RTC local contributing area
- - ld: range look direction angle
- - li: local incident angle
- - np: noise power (NESZ, per polarization)
- - gs: gamma-sigma ratio: sigma0 RTC / gamma0 RTC
- - sg: sigma-gamma ratio: gamma0 RTC / sigma0 ellipsoidal
- - wm: OCN product wind model; requires OCN scenes via argument `scenes_ocn`
- update: bool
- modify existing products so that only missing files are re-created?
-
- Returns
- -------
- str
- Either the time spent executing the function in seconds or 'Already processed - Skip!'
- """
- if compress is None:
- compress = 'LERC_ZSTD'
- if overviews is None:
- overviews = [2, 4, 9, 18, 36]
- ovr_resampling = 'AVERAGE'
- driver = 'COG'
- blocksize = 512
- dst_nodata_float = -9999.0
- dst_nodata_byte = 255
- vrt_nodata = 'nan' # was found necessary for proper calculation of statistics in QGIS
- vrt_options = {'VRTNodata': vrt_nodata}
-
- # determine processing timestamp and generate unique ID
- start_time = time.time()
- proc_time = datetime.now(timezone.utc)
- t = proc_time.isoformat().encode()
- product_id = generate_unique_id(encoded_str=t)
-
- src_ids, datasets_sar = get_datasets(scenes=scenes, datadir=datadir, extent=extent, epsg=epsg)
- if len(src_ids) == 0:
- print('None of the processed scenes overlap with the current tile {tile_id}'.format(tile_id=tile))
- return
-
- if annotation is not None:
- allowed = []
- for key in datasets_sar[0]:
- c1 = re.search('[gs]-lin', key)
- c2 = key in annotation
- c3 = key.startswith('np') and 'np' in annotation
- if c1 or c2 or c3:
- allowed.append(key)
- else:
- allowed = [key for key in datasets_sar[0].keys() if re.search('[gs]-lin', key)]
- annotation = []
- for item in ['em', 'id']:
- if item in annotation:
- allowed.append(item)
-
- # GDAL output bounds
- bounds = [extent['xmin'], extent['ymin'], extent['xmax'], extent['ymax']]
-
- ard_start, ard_stop = calc_product_start_stop(src_ids=src_ids, extent=extent, epsg=epsg)
- meta = {'mission': src_ids[0].sensor,
- 'mode': src_ids[0].meta['acquisition_mode'],
- 'ard_spec': product_type,
- 'polarization': {"['HH']": 'SH',
- "['VV']": 'SV',
- "['HH', 'HV']": 'DH',
- "['VV', 'VH']": 'DV'}[str(src_ids[0].polarizations)],
- 'start': ard_start,
- 'orbitnumber': src_ids[0].meta['orbitNumbers_abs']['start'],
- 'datatake': hex(src_ids[0].meta['frameNumber']).replace('x', '').upper(),
- 'tile': tile,
- 'id': product_id}
- meta_lower = dict((k, v.lower() if not isinstance(v, int) else v) for k, v in meta.items())
- skeleton_dir = '{mission}_{mode}_{ard_spec}__1S{polarization}_{start}_{orbitnumber:06}_{datatake:0>6}_{tile}_{id}'
- skeleton_files = '{mission}-{mode}-{ard_spec}-{start}-{orbitnumber:06}-{datatake:0>6}-{tile}-{suffix}.tif'
-
- ard_base = skeleton_dir.format(**meta)
- existing = finder(outdir, [ard_base.replace(product_id, '*')], foldermode=2)
- if len(existing) > 0:
- if not update:
- return 'Already processed - Skip!'
- else:
- ard_dir = existing[0]
- else:
- ard_dir = os.path.join(outdir, ard_base)
- os.makedirs(ard_dir, exist_ok=True)
- subdirectories = ['measurement', 'annotation', 'source', 'support']
- for subdirectory in subdirectories:
- os.makedirs(os.path.join(ard_dir, subdirectory), exist_ok=True)
-
- # prepare raster write options; https://gdal.org/drivers/raster/cog.html
- write_options_base = ['BLOCKSIZE={}'.format(blocksize),
- 'OVERVIEW_RESAMPLING={}'.format(ovr_resampling)]
- write_options = dict()
- for key in LERC_ERR_THRES:
- write_options[key] = write_options_base.copy()
- if compress is not None:
- entry = 'COMPRESS={}'.format(compress)
- write_options[key].append(entry)
- if compress.startswith('LERC'):
- entry = 'MAX_Z_ERROR={:f}'.format(LERC_ERR_THRES[key])
- write_options[key].append(entry)
-
- # create raster files: linear gamma0/sigma0 backscatter (-[vh|vv|hh|hv]-[gs]-lin.tif),
- # ellipsoidal incident angle (-ei.tif), gamma-to-sigma ratio (-gs.tif),
- # local contributing area (-lc.tif), local incident angle (-li.tif),
- # noise power images (-np-[vh|vv|hh|hv].tif)
- datasets_ard = dict()
- for key in list(datasets_sar[0].keys()):
- if key in ['dm', 'wm'] or key not in LERC_ERR_THRES.keys() or key not in allowed:
- # raster files for keys 'dm' and 'wm' are created later
- continue
-
- meta_lower['suffix'] = key
- outname_base = skeleton_files.format(**meta_lower)
- if re.search('[gs]-lin', key):
- subdir = 'measurement'
- else:
- subdir = 'annotation'
- outname = os.path.join(ard_dir, subdir, outname_base)
-
- if not os.path.isfile(outname):
- print(outname)
- images = [ds[key] for ds in datasets_sar]
- ras = None
- if len(images) > 1:
- ras = Raster(images, list_separate=False)
- source = ras.filename
- else:
- source = tempfile.NamedTemporaryFile(suffix='.vrt').name
- gdalbuildvrt(src=images[0], dst=source)
-
- # modify temporary VRT to make sure overview levels and resampling are properly applied
- vrt_add_overviews(vrt=source, overviews=overviews, resampling=ovr_resampling)
-
- options = {'format': driver, 'outputBounds': bounds,
- 'dstNodata': dst_nodata_float, 'multithread': multithread,
- 'creationOptions': write_options[key]}
-
- gdalwarp(src=source, dst=outname, **options)
- if ras is not None:
- ras.close()
- datasets_ard[key] = outname
-
- # define a reference raster from the annotation datasets and list all gamma0/sigma0 backscatter measurement rasters
- measure_tifs = [v for k, v in datasets_ard.items() if re.search('[gs]-lin', k)]
- ref_key = list(datasets_ard.keys())[0]
- ref_tif = datasets_ard[ref_key]
-
- # create data mask raster (-dm.tif)
- if 'dm' in allowed:
- if wbm is not None:
- if not config['dem_type'] == 'GETASSE30' and not os.path.isfile(wbm):
- raise FileNotFoundError('External water body mask could not be found: {}'.format(wbm))
-
- dm_path = ref_tif.replace(f'-{ref_key}.tif', '-dm.tif')
- if not os.path.isfile(dm_path):
- create_data_mask(outname=dm_path, datasets=datasets_sar, extent=extent, epsg=epsg,
- driver=driver, creation_opt=write_options['dm'],
- overviews=overviews, overview_resampling=ovr_resampling,
- dst_nodata=dst_nodata_byte, wbm=wbm, product_type=product_type)
- datasets_ard['dm'] = dm_path
-
- # create acquisition ID image raster (-id.tif)
- if 'id' in allowed:
- id_path = ref_tif.replace(f'-{ref_key}.tif', '-id.tif')
- if not os.path.isfile(id_path):
- create_acq_id_image(outname=id_path, ref_tif=ref_tif,
- datasets=datasets_sar, src_ids=src_ids,
- extent=extent, epsg=epsg, driver=driver,
- creation_opt=write_options['id'],
- overviews=overviews, dst_nodata=dst_nodata_byte)
- datasets_ard['id'] = id_path
-
- # create DEM (-em.tif)
- if dem_type is not None and 'em' in allowed:
- if kml is None:
- raise RuntimeError("If 'dem_type' is not None, `kml` needs to be defined.")
- em_path = ref_tif.replace(f'-{ref_key}.tif', '-em.tif')
- if not os.path.isfile(em_path):
- print(em_path)
- with Raster(ref_tif) as ras:
- tr = ras.res
- log_pyro = logging.getLogger('pyroSAR')
- level = log_pyro.level
- log_pyro.setLevel('NOTSET')
- dem.to_mgrs(dem_type=dem_type, dst=em_path, kml=kml,
- overviews=overviews, tile=tile, tr=tr,
- create_options=write_options['em'],
- pbar=False)
- log_pyro.setLevel(level)
- datasets_ard['em'] = em_path
-
- # create color composite VRT (-cc-[gs]-lin.vrt)
- if meta['polarization'] in ['DH', 'DV'] and len(measure_tifs) == 2:
- cc_path = re.sub('[hv]{2}', 'cc', measure_tifs[0]).replace('.tif', '.vrt')
- if not os.path.isfile(cc_path):
- create_rgb_vrt(outname=cc_path, infiles=measure_tifs,
- overviews=overviews, overview_resampling=ovr_resampling)
- key = re.search('cc-[gs]-lin', cc_path).group()
- datasets_ard[key] = cc_path
-
- # create log-scaled gamma0|sigma0 nought VRTs (-[vh|vv|hh|hv]-[gs]-log.vrt)
- fun = 'dB'
- args = {'fact': 10}
- scale = None
- for item in measure_tifs:
- target = item.replace('lin.tif', 'log.vrt')
- if not os.path.isfile(target):
- print(target)
- create_vrt(src=item, dst=target, fun=fun, scale=scale,
- args=args, options=vrt_options, overviews=overviews,
- overview_resampling=ovr_resampling)
- key = re.search('[hv]{2}-[gs]-log', target).group()
- datasets_ard[key] = target
-
- # create sigma nought RTC VRTs (-[vh|vv|hh|hv]-s-[lin|log].vrt)
- if 'gs' in allowed:
- gs_path = datasets_ard['gs']
- for item in measure_tifs:
- sigma0_rtc_lin = item.replace('g-lin.tif', 's-lin.vrt')
- sigma0_rtc_log = item.replace('g-lin.tif', 's-log.vrt')
-
- if not os.path.isfile(sigma0_rtc_lin):
- print(sigma0_rtc_lin)
- create_vrt(src=[item, gs_path], dst=sigma0_rtc_lin, fun='mul',
- relpaths=True, options=vrt_options, overviews=overviews,
- overview_resampling=ovr_resampling)
- key = re.search('[hv]{2}-s-lin', sigma0_rtc_lin).group()
- datasets_ard[key] = sigma0_rtc_lin
-
- if not os.path.isfile(sigma0_rtc_log):
- print(sigma0_rtc_log)
- create_vrt(src=sigma0_rtc_lin, dst=sigma0_rtc_log, fun=fun,
- scale=scale, options=vrt_options, overviews=overviews,
- overview_resampling=ovr_resampling, args=args)
- key = key.replace('lin', 'log')
- datasets_ard[key] = sigma0_rtc_log
-
- # create gamma nought RTC VRTs (-[vh|vv|hh|hv]-g-[lin|log].vrt)
- if 'sg' in allowed:
- sg_path = datasets_ard['sg']
- for item in measure_tifs:
- if not item.endswith('s-lin.tif'):
- continue
- gamma0_rtc_lin = item.replace('s-lin.tif', 'g-lin.vrt')
- gamma0_rtc_log = item.replace('s-lin.tif', 'g-log.vrt')
-
- if not os.path.isfile(gamma0_rtc_lin):
- print(gamma0_rtc_lin)
- create_vrt(src=[item, sg_path], dst=gamma0_rtc_lin, fun='mul',
- relpaths=True, options=vrt_options, overviews=overviews,
- overview_resampling=ovr_resampling)
- key = re.search('[hv]{2}-g-lin', gamma0_rtc_lin).group()
- datasets_ard[key] = gamma0_rtc_lin
-
- if not os.path.isfile(gamma0_rtc_log):
- print(gamma0_rtc_log)
- create_vrt(src=gamma0_rtc_lin, dst=gamma0_rtc_log, fun=fun,
- scale=scale, options=vrt_options, overviews=overviews,
- overview_resampling=ovr_resampling, args=args)
- key = key.replace('lin', 'log')
- datasets_ard[key] = gamma0_rtc_log
-
- # create backscatter wind model (-wm.tif)
- # and wind normalization VRT (-[vv|hh]-s-lin-wn.vrt)
- if 'wm' in annotation:
- wm = []
- for i, ds in enumerate(datasets_sar):
- if 'wm' in ds.keys():
- wm.append(ds['wm'])
- else:
- scene_base = os.path.basename(src_ids[i].scene)
- raise RuntimeError(f'could not find wind model product for scene {scene_base}')
-
- copol = 'VV' if 'VV' in src_ids[0].polarizations else 'HH'
- copol_sigma0_key = f'{copol.lower()}-s-lin'
- if copol_sigma0_key in datasets_ard.keys():
- copol_sigma0 = datasets_ard[copol_sigma0_key]
- wn_ard = re.sub(r's-lin\.(?:tif|vrt)', 's-lin-wn.vrt', copol_sigma0)
- else:
- copol_sigma0 = None
- wn_ard = None
-
- meta_lower['suffix'] = 'wm'
- outname_base = skeleton_files.format(**meta_lower)
- wm_ard = os.path.join(ard_dir, 'annotation', outname_base)
-
- gapfill = True if src_ids[0].product == 'GRD' else False
-
- wind_normalization(src=wm, dst_wm=wm_ard, dst_wn=wn_ard, measurement=copol_sigma0,
- gapfill=gapfill, bounds=bounds, epsg=epsg, driver=driver, creation_opt=write_options['wm'],
- dst_nodata=dst_nodata_float, multithread=multithread)
- datasets_ard['wm'] = wm_ard
- datasets_ard[f'{copol_sigma0_key}-wn'] = wn_ard
-
- # copy support files
- schema_dir = os.path.join(S1_NRB.__path__[0], 'validation', 'schemas')
- schemas = os.listdir(schema_dir)
- for schema in schemas:
- schema_in = os.path.join(schema_dir, schema)
- schema_out = os.path.join(ard_dir, 'support', schema)
- if not os.path.isfile(schema_out):
- print(schema_out)
- shutil.copy(schema_in, schema_out)
-
- # create metadata files in XML and (STAC) JSON formats
- start = datetime.strptime(ard_start, '%Y%m%dT%H%M%S')
- stop = datetime.strptime(ard_stop, '%Y%m%dT%H%M%S')
- meta = extract.meta_dict(config=config, target=ard_dir, src_ids=src_ids, sar_dir=datadir,
- proc_time=proc_time, start=start, stop=stop, compression=compress,
- product_type=product_type)
- ard_assets = sorted(sorted(list(datasets_ard.values()), key=lambda x: os.path.splitext(x)[1]),
- key=lambda x: os.path.basename(os.path.dirname(x)), reverse=True)
- if config['meta']['copy_original']:
- copy_src_meta(ard_dir=ard_dir, src_ids=src_ids)
- if 'OGC' in config['meta']['format']:
- xml.parse(meta=meta, target=ard_dir, assets=ard_assets, exist_ok=True)
- if 'STAC' in config['meta']['format']:
- stac.parse(meta=meta, target=ard_dir, assets=ard_assets, exist_ok=True)
- return str(round((time.time() - start_time), 2))
-
-
-def get_datasets(scenes, datadir, extent, epsg):
- """
- Collect processing output for a list of scenes.
- Reads metadata from all source SLC/GRD scenes, finds matching output files in `datadir`
- and filters both lists depending on the actual overlap of each SLC/GRD valid data coverage
- with the current MGRS tile geometry. If no output is found for any scene the function will raise an error.
- To obtain the extent of valid data coverage, first a binary
- mask raster file is created with the name `datamask.tif`, which is stored in the same folder as
- the processing output as found by :func:`~S1_NRB.snap.find_datasets`. Then, the boundary of this
- binary mask is computed and stored as `datamask.gpkg` (see function :func:`spatialist.vector.boundary`).
- If the provided `extent` does not overlap with this boundary, the output is discarded. This scenario
- might occur when the scene's geometry read from its metadata overlaps with the tile but the actual
- extent of data does not.
-
- Parameters
- ----------
- scenes: list[str]
- List of scenes to process. Either an individual scene or multiple, matching scenes (consecutive acquisitions).
- datadir: str
- The directory containing the SAR datasets processed from the source scenes using pyroSAR.
- The function will raise an error if the processing output cannot be found for all scenes in `datadir`.
- extent: dict
- Spatial extent of the MGRS tile, derived from a :class:`~spatialist.vector.Vector` object.
- epsg: int
- The coordinate reference system as an EPSG code.
-
- Returns
- -------
- ids: list[:class:`pyroSAR.drivers.ID`]
- List of :class:`~pyroSAR.drivers.ID` objects of all source SLC/GRD scenes that overlap with the current MGRS tile.
- datasets: list[dict]
- List of SAR processing output files that match each :class:`~pyroSAR.drivers.ID` object of `ids`.
- The format is a list of dictionaries per scene with keys as described by e.g. :func:`S1_NRB.snap.find_datasets`.
-
- See Also
- --------
- :func:`S1_NRB.snap.find_datasets`
- """
- ids = identify_many(scenes)
- datasets = []
- for i, _id in enumerate(ids):
- files = find_datasets(scene=_id.scene, outdir=datadir, epsg=epsg)
- if files is not None:
-
- base = os.path.splitext(os.path.basename(_id.scene))[0]
- ocn = re.sub('(?:SLC_|GRD[FHM])_1', 'OCN__2', base)[:-5]
- # allow 1 second tolerance
- s_start = int(ocn[31])
- s_stop = int(ocn[47])
- ocn_list = list(ocn)
- s = 1
- ocn_list[31] = f'[{s_start - s}{s_start}{s_start + s}]'
- ocn_list[47] = f'[{s_stop - s}{s_stop}{s_stop + s}]'
- ocn = ''.join(ocn_list)
- ocn_match = finder(target=datadir, matchlist=[ocn], regex=True, foldermode=2)
- if len(ocn_match) > 0:
- cmod = os.path.join(ocn_match[0], 'owiNrcsCmod.tif')
- if os.path.isfile(cmod):
- files['wm'] = cmod
-
- datasets.append(files)
- else:
- base = os.path.basename(_id.scene)
- raise RuntimeError(f'cannot find processing output for scene {base}')
-
- i = 0
- while i < len(datasets):
- measurements = [datasets[i][x] for x in datasets[i].keys() if re.search('[gs]-lin', x)]
- dm_ras = os.path.join(os.path.dirname(measurements[0]), 'datamask.tif')
- dm_vec = dm_ras.replace('.tif', '.gpkg')
-
- if not os.path.isfile(dm_ras):
- with Raster(measurements[0]) as ras:
- arr = ras.array()
- mask = ~np.isnan(arr)
- del arr
- # remove scene if file does not contain valid data
- if len(mask[mask == 1]) == 0:
- del ids[i], datasets[i]
- continue
- with vectorize(target=mask, reference=ras) as vec:
- with boundary(vec, expression="value=1") as bounds:
- if not os.path.isfile(dm_ras):
- rasterize(vectorobject=bounds, reference=ras, outname=dm_ras)
- if not os.path.isfile(dm_vec):
- bounds.write(outfile=dm_vec)
- del mask
- if not os.path.isfile(dm_vec):
- with Raster(dm_ras) as ras:
- mask = ras.array().astype('bool')
- # remove scene if file does not contain valid data
- if len(mask[mask == 1]) == 0:
- del ids[i], datasets[i]
- continue
- with vectorize(target=mask, reference=ras) as vec:
- boundary(vec, expression="value=1", outname=dm_vec)
- del mask
- with Vector(dm_vec) as bounds:
- with bbox(extent, epsg) as tile_geom:
- inter = intersect(bounds, tile_geom)
- if inter is None:
- del ids[i]
- del datasets[i]
- else:
- # Add dm_ras to the datasets if it overlaps with the current tile
- datasets[i]['datamask'] = dm_ras
- i += 1
- inter.close()
- return ids, datasets
-
-
-def create_vrt(src, dst, fun, relpaths=False, scale=None, offset=None, args=None,
- options=None, overviews=None, overview_resampling=None):
- """
- Creates a VRT file for the specified source dataset(s) and adds a pixel function that should be applied on the fly
- when opening the VRT file.
-
- Parameters
- ----------
- src: str or list[str]
- The input dataset(s).
- dst: str
- The output dataset.
- fun: str
- A `PixelFunctionType` that should be applied on the fly when opening the VRT file. The function is applied to a
- band that derives its pixel information from the source bands. A list of possible options can be found here:
- https://gdal.org/drivers/raster/vrt.html#default-pixel-functions.
- Furthermore, the option 'decibel' can be specified, which will implement a custom pixel function that uses
- Python code for decibel conversion (10*log10).
- relpaths: bool
- Should all `SourceFilename` XML elements with attribute `@relativeToVRT="0"` be updated to be paths relative to
- the output VRT file? Default is False.
- scale: int or None
- The scale that should be applied when computing “real” pixel values from scaled pixel values on a raster band.
- Will be ignored if `fun='decibel'`.
- offset: float or None
- The offset that should be applied when computing “real” pixel values from scaled pixel values on a raster band.
- Will be ignored if `fun='decibel'`.
- args: dict or None
- arguments for `fun` passed as `PixelFunctionArguments`. Requires GDAL>=3.5 to be read.
- options: dict or None
- Additional parameters passed to `gdal.BuildVRT`.
- overviews: list[int] or None
- Internal overview levels to be created for each raster file.
- overview_resampling: str or None
- Resampling method for overview levels.
-
- Examples
- --------
- linear gamma0 backscatter as input:
-
- >>> src = 's1a-iw-nrb-20220601t052704-043465-0530a1-32tpt-vh-g-lin.tif'
-
- decibel scaling I:
- use `log10` pixel function and additional `Scale` parameter.
- Known to display well in QGIS, but `Scale` is ignored when reading array in Python.
-
- >>> dst = src.replace('-lin.tif', '-log1.vrt')
- >>> create_vrt(src=src, dst=dst, fun='log10', scale=10)
-
- decibel scaling II:
- use custom Python pixel function. Requires additional environment variable GDAL_VRT_ENABLE_PYTHON set to YES.
-
- >>> dst = src.replace('-lin.tif', '-log2.vrt')
- >>> create_vrt(src=src, dst=dst, fun='decibel')
-
- decibel scaling III:
- use `dB` pixel function with additional `PixelFunctionArguments`. Works best but requires GDAL>=3.5.
-
- >>> dst = src.replace('-lin.tif', '-log3.vrt')
- >>> create_vrt(src=src, dst=dst, fun='dB', args={'fact': 10})
- """
- options = {} if options is None else options
- gdalbuildvrt(src=src, dst=dst, **options)
- tree = etree.parse(dst)
- root = tree.getroot()
- band = tree.find('VRTRasterBand')
- band.attrib['subClass'] = 'VRTDerivedRasterBand'
-
- if fun == 'decibel':
- pxfun_language = etree.SubElement(band, 'PixelFunctionLanguage')
- pxfun_language.text = 'Python'
- pxfun_type = etree.SubElement(band, 'PixelFunctionType')
- pxfun_type.text = fun
- pxfun_code = etree.SubElement(band, 'PixelFunctionCode')
- pxfun_code.text = etree.CDATA("""
- import numpy as np
- def decibel(in_ar, out_ar, xoff, yoff, xsize, ysize, raster_xsize, raster_ysize, buf_radius, gt, **kwargs):
- np.multiply(np.log10(in_ar[0], where=in_ar[0]>0.0, out=out_ar, dtype='float32'), 10.0, out=out_ar, dtype='float32')
- """)
- else:
- pixfun_type = etree.SubElement(band, 'PixelFunctionType')
- pixfun_type.text = fun
- if args is not None:
- arg = etree.SubElement(band, 'PixelFunctionArguments')
- for key, value in args.items():
- arg.attrib[key] = str(value)
- if scale is not None:
- sc = etree.SubElement(band, 'Scale')
- sc.text = str(scale)
- if offset is not None:
- off = etree.SubElement(band, 'Offset')
- off.text = str(offset)
-
- if any([overviews, overview_resampling]) is not None:
- ovr = tree.find('OverviewList')
- if ovr is None:
- ovr = etree.SubElement(root, 'OverviewList')
- if overview_resampling is not None:
- ovr.attrib['resampling'] = overview_resampling.lower()
- if overviews is not None:
- ov = str(overviews)
- for x in ['[', ']', ',']:
- ov = ov.replace(x, '')
- ovr.text = ov
-
- if relpaths:
- srcfiles = tree.xpath('//SourceFilename[@relativeToVRT="0"]')
- for srcfile in srcfiles:
- repl = os.path.relpath(srcfile.text, start=os.path.dirname(dst))
- repl = repl.replace('\\', '/')
- srcfile.text = repl
- srcfile.attrib['relativeToVRT'] = '1'
-
- etree.indent(root)
- tree.write(dst, pretty_print=True, xml_declaration=False, encoding='utf-8')
-
-
-def create_rgb_vrt(outname, infiles, overviews, overview_resampling):
- """
- Creation of the color composite VRT file.
-
- Parameters
- ----------
- outname: str
- Full path to the output VRT file.
- infiles: list[str]
- A list of paths pointing to the linear scaled measurement backscatter files.
- overviews: list[int]
- Internal overview levels to be defined for the created VRT file.
- overview_resampling: str
- Resampling method applied to overview pyramids.
- """
- print(outname)
-
- # make sure order is right and co-polarization (VV or HH) is first
- pols = [re.search('[hv]{2}', os.path.basename(f)).group() for f in infiles]
- if pols[1] in ['vv', 'hh']:
- infiles.reverse()
- pols.reverse()
-
- # format overview levels
- ov = str(overviews)
- for x in ['[', ']', ',']:
- ov = ov.replace(x, '')
-
- # create VRT file and change its content
- gdalbuildvrt(src=infiles, dst=outname, separate=True)
-
- tree = etree.parse(outname)
- root = tree.getroot()
- srs = tree.find('SRS').text
- geotrans = tree.find('GeoTransform').text
- bands = tree.findall('VRTRasterBand')
- vrt_nodata = bands[0].find('NoDataValue').text
- complex_src = [band.find('ComplexSource') for band in bands]
- for cs in complex_src:
- cs.remove(cs.find('NODATA'))
-
- new_band = etree.SubElement(root, 'VRTRasterBand',
- attrib={'dataType': 'Float32', 'band': '3',
- 'subClass': 'VRTDerivedRasterBand'})
- new_band_na = etree.SubElement(new_band, 'NoDataValue')
- new_band_na.text = 'nan'
- pxfun_type = etree.SubElement(new_band, 'PixelFunctionType')
- pxfun_type.text = 'mul'
- for cs in complex_src:
- new_band.append(deepcopy(cs))
-
- src = new_band.findall('ComplexSource')[1]
- fname = src.find('SourceFilename')
- fname_old = fname.text
- src_attr = src.find('SourceProperties').attrib
- fname.text = etree.CDATA("""
-
- {srs}
- {geotrans}
-
- {vrt_nodata}
- {px_fun}
-
- {fname}
- 1
-
-
-
-
-
- {ov}
-
- """.format(rasterxsize=src_attr['RasterXSize'], rasterysize=src_attr['RasterYSize'], srs=srs, geotrans=geotrans,
- dtype=src_attr['DataType'], px_fun='inv', fname=fname_old, vrt_nodata=vrt_nodata,
- blockxsize=src_attr['BlockXSize'], blockysize=src_attr['BlockYSize'],
- ov_resampling=overview_resampling.lower(), ov=ov))
-
- bands = tree.findall('VRTRasterBand')
- for band, col in zip(bands, ['Red', 'Green', 'Blue']):
- color = etree.Element('ColorInterp')
- color.text = col
- band.insert(0, color)
-
- ovr = etree.SubElement(root, 'OverviewList', attrib={'resampling': overview_resampling.lower()})
- ovr.text = ov
-
- etree.indent(root)
- tree.write(outname, pretty_print=True, xml_declaration=False, encoding='utf-8')
-
-
-def calc_product_start_stop(src_ids, extent, epsg):
- """
- Calculates the start and stop times of the ARD product.
- The geolocation grid points including their azimuth time information are extracted first from the metadata of each
- source SLC. These grid points are then used to interpolate the azimuth time for the lower right and upper left
- (ascending) or upper right and lower left (descending) corners of the MGRS tile extent.
-
- Parameters
- ----------
- src_ids: list[pyroSAR.drivers.ID]
- List of :class:`~pyroSAR.drivers.ID` objects of all source SLC scenes that overlap with the current MGRS tile.
- extent: dict
- Spatial extent of the MGRS tile, derived from a :class:`~spatialist.vector.Vector` object.
- epsg: int
- The coordinate reference system as an EPSG code.
-
- Returns
- -------
- tuple[str]
-
- - Start time of the ARD product formatted as `%Y%m%dT%H%M%S` in UTC.
- - Stop time of the ARD product formatted as `%Y%m%dT%H%M%S` in UTC.
- """
- with bbox(extent, epsg) as tile_geom:
- tile_geom.reproject(4326)
- ext = tile_geom.extent
- ul = (ext['xmin'], ext['ymax'])
- ur = (ext['xmax'], ext['ymax'])
- lr = (ext['xmax'], ext['ymin'])
- ll = (ext['xmin'], ext['ymin'])
- tile_geom = None
-
- slc_dict = {}
- for i, sid in enumerate(src_ids):
- uid = os.path.basename(sid.scene).split('.')[0][-4:]
- slc_dict[uid] = get_src_meta(sid)
- slc_dict[uid]['sid'] = sid
-
- uids = list(slc_dict.keys())
-
- for uid in uids:
- t = find_in_annotation(annotation_dict=slc_dict[uid]['annotation'],
- pattern='.//geolocationGridPoint/azimuthTime')
- swaths = t.keys()
- y = find_in_annotation(annotation_dict=slc_dict[uid]['annotation'], pattern='.//geolocationGridPoint/latitude')
- x = find_in_annotation(annotation_dict=slc_dict[uid]['annotation'], pattern='.//geolocationGridPoint/longitude')
- t_flat = np.asarray([datetime.fromisoformat(item).timestamp() for sublist in [t[swath] for swath in swaths]
- for item in sublist])
- y_flat = np.asarray([float(item) for sublist in [y[swath] for swath in swaths] for item in sublist])
- x_flat = np.asarray([float(item) for sublist in [x[swath] for swath in swaths] for item in sublist])
- g = np.asarray([(x, y) for x, y in zip(x_flat, y_flat)])
- slc_dict[uid]['az_time'] = t_flat
- slc_dict[uid]['gridpts'] = g
-
- if len(uids) == 2:
- starts = [datetime.strptime(slc_dict[key]['sid'].start, '%Y%m%dT%H%M%S') for key in slc_dict.keys()]
- if starts[0] > starts[1]:
- az_time = np.concatenate([slc_dict[uids[1]]['az_time'], slc_dict[uids[0]]['az_time']])
- gridpts = np.concatenate([slc_dict[uids[1]]['gridpts'], slc_dict[uids[0]]['gridpts']])
- else:
- az_time = np.concatenate([slc_dict[key]['az_time'] for key in slc_dict.keys()])
- gridpts = np.concatenate([slc_dict[key]['gridpts'] for key in slc_dict.keys()])
- else:
- az_time = slc_dict[uids[0]]['az_time']
- gridpts = slc_dict[uids[0]]['gridpts']
-
- if slc_dict[uids[0]]['sid'].orbit == 'A':
- coord1 = lr
- coord2 = ul
- else:
- coord1 = ur
- coord2 = ll
-
- method = 'linear'
- res = [griddata(gridpts, az_time, coord1, method=method),
- griddata(gridpts, az_time, coord2, method=method)]
-
- min_start = min([datetime.strptime(slc_dict[uid]['sid'].start, '%Y%m%dT%H%M%S') for uid in uids])
- max_stop = max([datetime.strptime(slc_dict[uid]['sid'].stop, '%Y%m%dT%H%M%S') for uid in uids])
- res_t = []
- for i, r in enumerate(res):
- if np.isnan(r):
- if i == 0:
- res_t.append(min_start)
- else:
- res_t.append(max_stop)
- else:
- res_t.append(datetime.fromtimestamp(float(r)))
-
- start = datetime.strftime(res_t[0], '%Y%m%dT%H%M%S')
- stop = datetime.strftime(res_t[1], '%Y%m%dT%H%M%S')
-
- return start, stop
-
-
-def create_data_mask(outname, datasets, extent, epsg, driver, creation_opt,
- overviews, overview_resampling, dst_nodata, product_type, wbm=None):
- """
- Creation of the Data Mask image.
-
- Parameters
- ----------
- outname: str
- Full path to the output data mask file.
- datasets: list[dict]
- List of processed output files that match the source scenes and overlap with the current MGRS tile.
- An error will be thrown if not all datasets contain a key `datamask`.
- The function will return without an error if not all datasets contain a key `dm`.
- extent: dict
- Spatial extent of the MGRS tile, derived from a :class:`~spatialist.vector.Vector` object.
- epsg: int
- The coordinate reference system as an EPSG code.
- driver: str
- GDAL driver to use for raster file creation.
- creation_opt: list[str]
- GDAL creation options to use for raster file creation. Should match specified GDAL driver.
- overviews: list[int]
- Internal overview levels to be created for each raster file.
- overview_resampling: str
- Resampling method for overview levels.
- dst_nodata: int or str
- Nodata value to write to the output raster.
- product_type: str
- The type of ARD product that is being created. Either 'NRB' or 'ORB'.
- wbm: str or None
- Path to a water body mask file with the dimensions of an MGRS tile. Optional if `product_type='NRB', mandatory
- if `product_type='ORB'`.
- """
- measurement_keys = [x for x in datasets[0].keys() if re.search('[gs]-lin', x)]
- measurement = [scene[measurement_keys[0]] for scene in datasets]
- datamask = [scene['datamask'] for scene in datasets]
- ls = []
- for scene in datasets:
- if 'dm' in scene:
- ls.append(scene['dm'])
- else:
- return # do not create a data mask if not all scenes have a layover-shadow mask
- print(outname)
- if product_type == 'NRB':
- dm_bands = [{'arr_val': 0, 'name': 'not layover, nor shadow'},
- {'arr_val': 1, 'name': 'layover'},
- {'arr_val': 2, 'name': 'shadow'},
- # {'arr_val': 3, 'name': 'layover and shadow'}, # just for context, not used as an individual band
- {'arr_val': 4, 'name': 'ocean water'}]
- elif product_type == 'ORB':
- if wbm is None:
- raise RuntimeError('Water body mask is required for ORB products')
- dm_bands = [{'arr_val': [0, 1, 2], 'name': 'land'},
- {'arr_val': 4, 'name': 'ocean water'}]
- else:
- raise RuntimeError(f'Unknown product type: {product_type}')
-
- tile_bounds = [extent['xmin'], extent['ymin'], extent['xmax'], extent['ymax']]
-
- vrt_ls = '/vsimem/' + os.path.dirname(outname) + 'ls.vrt'
- vrt_valid = '/vsimem/' + os.path.dirname(outname) + 'valid.vrt'
- vrt_measurement = '/vsimem/' + os.path.dirname(outname) + 'measurement.vrt'
- gdalbuildvrt(src=ls, dst=vrt_ls, outputBounds=tile_bounds, void=False)
- gdalbuildvrt(src=datamask, dst=vrt_valid, outputBounds=tile_bounds, void=False)
- gdalbuildvrt(src=measurement, dst=vrt_measurement, outputBounds=tile_bounds, void=False)
-
- with Raster(vrt_ls) as ras_ls:
- with bbox(extent, crs=epsg) as tile_vec:
- ras_ls_res = ras_ls.res
- rows = ras_ls.rows
- cols = ras_ls.cols
- geotrans = ras_ls.raster.GetGeoTransform()
- proj = ras_ls.raster.GetProjection()
- arr_dm = ras_ls.array()
-
- # Add Water Body Mask
- if wbm is not None:
- with Raster(wbm) as ras_wbm:
- ras_wbm_cols = ras_wbm.cols
- cols_ratio = ras_wbm_cols / cols
- if cols_ratio > 1:
- res = int(ras_ls_res[0])
- wbm_lowres = wbm.replace('.tif', f'_{res}m.vrt')
- options = {'xRes': res, 'yRes': res,
- 'resampleAlg': 'mode'}
- gdalbuildvrt(src=wbm, dst=wbm_lowres, **options)
- with Raster(wbm_lowres) as ras_wbm_lowres:
- arr_wbm = ras_wbm_lowres.array()
- else:
- arr_wbm = ras_wbm.array()
- out_arr = np.where((arr_wbm == 1), 4, arr_dm)
- del arr_wbm
- else:
- out_arr = arr_dm
- del dm_bands[3]
- del arr_dm
-
- # Extend the shadow class of the data mask with nodata values from backscatter data and create final array
- with Raster(vrt_valid)[tile_vec] as ras_valid:
- with Raster(vrt_measurement)[tile_vec] as ras_measurement:
- arr_valid = ras_valid.array()
- arr_measurement = ras_measurement.array()
-
- out_arr = np.nan_to_num(out_arr)
- out_arr = np.where(((arr_valid == 1) & (np.isnan(arr_measurement)) & (out_arr != 4)), 2,
- out_arr)
- out_arr[np.isnan(arr_valid)] = dst_nodata
- del arr_measurement
- del arr_valid
-
- outname_tmp = '/vsimem/' + os.path.basename(outname) + '.vrt'
- gdriver = gdal.GetDriverByName('GTiff')
- ds_tmp = gdriver.Create(outname_tmp, rows, cols, len(dm_bands), gdal.GDT_Byte,
- options=['ALPHA=UNSPECIFIED', 'PHOTOMETRIC=MINISWHITE'])
- gdriver = None
- ds_tmp.SetGeoTransform(geotrans)
- ds_tmp.SetProjection(proj)
-
- for i, _dict in enumerate(dm_bands):
- band = ds_tmp.GetRasterBand(i + 1)
- arr_val = _dict['arr_val']
- b_name = _dict['name']
-
- arr = np.full((rows, cols), 0)
- arr[out_arr == dst_nodata] = dst_nodata
- if arr_val in [0, 4]:
- arr[out_arr == arr_val] = 1
- elif arr_val in [1, 2]:
- arr[(out_arr == arr_val) | (out_arr == 3)] = 1
- elif arr_val == [0, 1, 2]:
- arr[out_arr != 4] = 1
-
- arr = arr.astype('uint8')
- band.WriteArray(arr)
- band.SetNoDataValue(dst_nodata)
- band.SetDescription(b_name)
- band.FlushCache()
- band = None
- del arr
-
- ds_tmp.SetMetadataItem('TIFFTAG_DATETIME', strftime('%Y:%m:%d %H:%M:%S', gmtime()))
- ds_tmp.BuildOverviews(overview_resampling, overviews)
- outDataset_cog = gdal.GetDriverByName(driver).CreateCopy(outname, ds_tmp, strict=1, options=creation_opt)
- outDataset_cog = None
- ds_tmp = None
- tile_vec = None
-
-
-def create_acq_id_image(outname, ref_tif, datasets, src_ids, extent,
- epsg, driver, creation_opt, overviews, dst_nodata):
- """
- Creation of the Acquisition ID image.
-
- Parameters
- ----------
- outname: str
- Full path to the output data mask file.
- ref_tif: str
- Full path to any GeoTIFF file of the ARD product.
- datasets: list[dict]
- List of processed output files that match the source SLC scenes and overlap with the current MGRS tile.
- src_ids: list[pyroSAR.drivers.ID]
- List of :class:`~pyroSAR.drivers.ID` objects of all source SLC scenes that overlap with the current MGRS tile.
- extent: dict
- Spatial extent of the MGRS tile, derived from a :class:`~spatialist.vector.Vector` object.
- epsg: int
- The CRS used for the ARD product; provided as an EPSG code.
- driver: str
- GDAL driver to use for raster file creation.
- creation_opt: list[str]
- GDAL creation options to use for raster file creation. Should match specified GDAL driver.
- overviews: list[int]
- Internal overview levels to be created for each raster file.
- dst_nodata: int or str
- Nodata value to write to the output raster.
- """
- print(outname)
- src_scenes = [sid.scene for sid in src_ids]
- # If there are two source scenes, make sure that the order of acquisitions in all lists is correct!
- if len(src_scenes) > 1:
- if not len(src_scenes) == 2 and len(datasets) == 2:
- raise RuntimeError('expected lists `src_scenes` and `valid_mask_list` to be of length 2; length is '
- '{} and {} respectively'.format(len(src_scenes), len(datasets)))
- starts_src = [datetime.strptime(identify(f).start, '%Y%m%dT%H%M%S') for f in src_scenes]
- start_valid = [re.search('[0-9]{8}T[0-9]{6}', os.path.basename(x)).group() for x in src_scenes]
- start_valid = [datetime.strptime(x, '%Y%m%dT%H%M%S') for x in start_valid]
- if starts_src[0] > starts_src[1]:
- src_scenes.reverse()
- starts_src.reverse()
- if start_valid[0] != starts_src[0]:
- datasets.reverse()
- if start_valid[0] != starts_src[0]:
- raise RuntimeError('failed to match order of lists `src_scenes` and `valid_mask_list`')
-
- tile_bounds = [extent['xmin'], extent['ymin'], extent['xmax'], extent['ymax']]
-
- arr_list = []
- for dataset in datasets:
- vrt_valid = '/vsimem/' + os.path.dirname(outname) + 'mosaic.vrt'
- gdalbuildvrt(src=dataset['datamask'], dst=vrt_valid, outputBounds=tile_bounds, void=False)
- with bbox(extent, crs=epsg) as tile_vec:
- with Raster(vrt_valid)[tile_vec] as vrt_ras:
- vrt_arr = vrt_ras.array()
- arr_list.append(vrt_arr)
- del vrt_arr
- tile_vec = None
-
- src_scenes_clean = [os.path.basename(src).replace('.zip', '').replace('.SAFE', '') for src in src_scenes]
- tag = '{{"{src1}": 1}}'.format(src1=src_scenes_clean[0])
- out_arr = np.full(arr_list[0].shape, dst_nodata)
- out_arr[arr_list[0] == 1] = 1
- if len(arr_list) == 2:
- out_arr[arr_list[1] == 1] = 2
- tag = '{{"{src1}": 1, "{src2}": 2}}'.format(src1=src_scenes_clean[0], src2=src_scenes_clean[1])
-
- creation_opt.append('TIFFTAG_IMAGEDESCRIPTION={}'.format(tag))
- with Raster(ref_tif) as ref_ras:
- ref_ras.write(outname, format=driver, array=out_arr.astype('uint8'), nodata=dst_nodata, overwrite=True,
- overviews=overviews, options=creation_opt)
-
-
-def wind_normalization(src, dst_wm, dst_wn, measurement, gapfill, bounds, epsg, driver, creation_opt, dst_nodata,
- multithread, resolution=915):
- """
- Create wind normalization layers. A wind model annotation layer is created and optionally
- a wind normalization VRT.
-
- Parameters
- ----------
- src: list[str]
- A list of OCN products as prepared by :func:`S1_NRB.ocn.extract`
- dst_wm: str
- The name of the wind model layer in the ARD product
- dst_wn: str or None
- The name of the wind normalization VRT. If None, no VRT will be created.
- Requires `measurement` to point to a file.
- measurement: str or None
- The name of the measurement file used for wind normalization in `dst_wn`.
- If None, no wind normalization VRT will be created.
- gapfill: bool
- Perform additional gap filling (:func:`S1_NRB.ocn.gapfill`)?
- This is recommended if the Level-1 source product of `measurement` is GRD
- in which case gaps are introduced between subsequently acquired scenes.
- bounds: list[float]
- the bounds of the MGRS tile
- epsg: int
- The EPSG code of the MGRS tile
- driver: str
- GDAL driver to use for raster file creation.
- creation_opt: list[str]
- GDAL creation options to use for raster file creation. Should match specified GDAL driver.
- dst_nodata: float
- Nodata value to write to the output raster.
- multithread: bool
- Should `gdalwarp` use multithreading?
- resolution: int, optional
- The target pixel resolution in meters. 915 is chosen as default because it is closest
- to the OCN product resolution (1000) and still fits into the MGRS bounds
- (``109800 % 915 == 0``).
-
- Returns
- -------
-
- """
- if len(src) > 1:
- cmod_mosaic = tempfile.NamedTemporaryFile(suffix='.tif').name
- gdalwarp(src=src, dst=cmod_mosaic)
- if gapfill:
- cmod_geo = tempfile.NamedTemporaryFile(suffix='.tif').name
- ocn.gapfill(src=cmod_mosaic, dst=cmod_geo, md=1, si=1)
- else:
- cmod_geo = cmod_mosaic
- else:
- cmod_geo = src[0]
-
- if not os.path.isfile(dst_wm):
- print(dst_wm)
- gdalwarp(src=cmod_geo,
- dst=dst_wm,
- outputBounds=bounds,
- dstSRS=f'EPSG:{epsg}',
- xRes=resolution, yRes=resolution,
- resampleAlg='bilinear',
- format=driver,
- dstNodata=dst_nodata,
- multithread=multithread,
- creationOptions=creation_opt)
-
- if dst_wn is not None and measurement is not None:
- if not os.path.isfile(dst_wn):
- print(dst_wn)
- with Raster(measurement) as ras:
- xres, yres = ras.res
- create_vrt(src=[measurement, dst_wm], dst=dst_wn, fun='div',
- options={'xRes': xres, 'yRes': yres})
+import os
+import re
+import time
+import shutil
+import logging
+import tempfile
+from datetime import datetime, timezone
+import numpy as np
+from lxml import etree
+from time import gmtime, strftime
+from copy import deepcopy
+from scipy.interpolate import griddata
+from osgeo import gdal
+from spatialist import Raster, Vector, vectorize, boundary, bbox, intersect, rasterize
+from spatialist.auxil import gdalwarp, gdalbuildvrt
+from spatialist.ancillary import finder
+from pyroSAR import identify, identify_many
+import S1_NRB
+from S1_NRB import dem, ocn
+from S1_NRB.metadata import extract, xml, stac
+from S1_NRB.metadata.mapping import LERC_ERR_THRES
+from S1_NRB.ancillary import generate_unique_id, vrt_add_overviews
+from S1_NRB.metadata.extract import copy_src_meta, get_src_meta, find_in_annotation
+from S1_NRB.snap import find_datasets
+
+
+def format(config, product_type, scenes, datadir, outdir, tile, extent, epsg, wbm=None, dem_type=None, multithread=True,
+ compress=None, overviews=None, kml=None, annotation=None, update=False):
+ """
+ Finalizes the generation of Sentinel-1 Analysis Ready Data (ARD) products after SAR processing has finished.
+ This includes the following:
+
+ - Creating all measurement and annotation datasets in Cloud Optimized GeoTIFF (COG) format
+ - Creating additional annotation datasets in Virtual Raster Tile (VRT) format
+ - Applying the ARD product directory structure & naming convention
+ - Generating metadata in XML and JSON formats for the ARD product as well as source SLC datasets
+
+ Parameters
+ ----------
+ config: dict
+ Dictionary of the parsed config parameters for the current process.
+ product_type: str
+ The type of ARD product to be generated. Options: 'NRB' or 'ORB'.
+ scenes: list[str]
+ List of scenes to process. Either a single scene or multiple, matching scenes (consecutive acquisitions).
+ All scenes are expected to overlap with `extent` and an error will be thrown if the processing output
+ cannot be found for any of the scenes.
+ datadir: str
+ The directory containing the SAR datasets processed from the source scenes using pyroSAR.
+ outdir: str
+ The directory to write the final files to.
+ tile: str
+ ID of an MGRS tile.
+ extent: dict
+ Spatial extent of the MGRS tile, derived from a :class:`~spatialist.vector.Vector` object.
+ epsg: int
+ The CRS used for the ARD product; provided as an EPSG code.
+ wbm: str or None
+ Path to a water body mask file with the dimensions of an MGRS tile.
+ dem_type: str or None
+ if defined, a DEM layer will be added to the product. The suffix `em` (elevation model) is used.
+ Default `None`: do not add a DEM layer.
+ multithread: bool
+ Should `gdalwarp` use multithreading? Default is True. The number of threads used, can be adjusted in the
+ `config.ini` file with the parameter `gdal_threads`.
+ compress: str or None
+ Compression algorithm to use. See https://gdal.org/drivers/raster/gtiff.html#creation-options for options.
+ Defaults to 'LERC_DEFLATE'.
+ overviews: list[int] or None
+ Internal overview levels to be created for each GeoTIFF file. Defaults to [2, 4, 9, 18, 36]
+ kml: str or None
+ The KML file containing the MGRS tile geometries. Only needs to be defined if `dem_type!=None`.
+ annotation: list[str] or None
+ an optional list to select the annotation layers. Default `None`: create all layers if the
+ source products contain the required input layers. Options:
+
+ - dm: data mask (four masks: not layover not shadow, layover, shadow, ocean water)
+ - ei: ellipsoidal incident angle
+ - em: digital elevation model
+ - id: acquisition ID image (source scene ID per pixel)
+ - lc: RTC local contributing area
+ - ld: range look direction angle
+ - li: local incident angle
+ - np: noise power (NESZ, per polarization)
+ - gs: gamma-sigma ratio: sigma0 RTC / gamma0 RTC
+ - sg: sigma-gamma ratio: gamma0 RTC / sigma0 ellipsoidal
+ - wm: OCN product wind model; requires OCN scenes via argument `scenes_ocn`
+ update: bool
+ modify existing products so that only missing files are re-created?
+
+ Returns
+ -------
+ str
+ Either the time spent executing the function in seconds or 'Already processed - Skip!'
+ """
+ if compress is None:
+ compress = 'LERC_ZSTD'
+ if overviews is None:
+ overviews = [2, 4, 9, 18, 36]
+ ovr_resampling = 'AVERAGE'
+ driver = 'COG'
+ blocksize = 512
+ dst_nodata_float = -9999.0
+ dst_nodata_byte = 255
+ vrt_nodata = 'nan' # was found necessary for proper calculation of statistics in QGIS
+ vrt_options = {'VRTNodata': vrt_nodata}
+
+ # determine processing timestamp and generate unique ID
+ start_time = time.time()
+ proc_time = datetime.now(timezone.utc)
+ t = proc_time.isoformat().encode()
+ product_id = generate_unique_id(encoded_str=t)
+
+ src_ids, datasets_sar = get_datasets(scenes=scenes, datadir=datadir, extent=extent, epsg=epsg)
+ if len(src_ids) == 0:
+ print('None of the processed scenes overlap with the current tile {tile_id}'.format(tile_id=tile))
+ return
+
+ if annotation is not None:
+ allowed = []
+ for key in datasets_sar[0]:
+ c1 = re.search('[gs]-lin', key)
+ c2 = key in annotation
+ c3 = key.startswith('np') and 'np' in annotation
+ if c1 or c2 or c3:
+ allowed.append(key)
+ else:
+ allowed = [key for key in datasets_sar[0].keys() if re.search('[gs]-lin', key)]
+ annotation = []
+ for item in ['em', 'id']:
+ if item in annotation:
+ allowed.append(item)
+
+ # GDAL output bounds
+ bounds = [extent['xmin'], extent['ymin'], extent['xmax'], extent['ymax']]
+
+ ard_start, ard_stop = calc_product_start_stop(src_ids=src_ids, extent=extent, epsg=epsg)
+ meta = {'mission': src_ids[0].sensor,
+ 'mode': src_ids[0].meta['acquisition_mode'],
+ 'ard_spec': product_type,
+ 'polarization': {"['HH']": 'SH',
+ "['VV']": 'SV',
+ "['HH', 'HV']": 'DH',
+ "['VV', 'VH']": 'DV'}[str(src_ids[0].polarizations)],
+ 'start': ard_start,
+ 'orbitnumber': src_ids[0].meta['orbitNumbers_abs']['start'],
+ 'datatake': hex(src_ids[0].meta['frameNumber']).replace('x', '').upper(),
+ 'tile': tile,
+ 'id': product_id}
+ meta_lower = dict((k, v.lower() if not isinstance(v, int) else v) for k, v in meta.items())
+ skeleton_dir = '{mission}_{mode}_{ard_spec}__1S{polarization}_{start}_{orbitnumber:06}_{datatake:0>6}_{tile}_{id}'
+ skeleton_files = '{mission}-{mode}-{ard_spec}-{start}-{orbitnumber:06}-{datatake:0>6}-{tile}-{suffix}.tif'
+
+ ard_base = skeleton_dir.format(**meta)
+ existing = finder(outdir, [ard_base.replace(product_id, '*')], foldermode=2)
+ if len(existing) > 0:
+ if not update:
+ return 'Already processed - Skip!'
+ else:
+ ard_dir = existing[0]
+ else:
+ ard_dir = os.path.join(outdir, ard_base)
+ os.makedirs(ard_dir, exist_ok=True)
+ subdirectories = ['measurement', 'annotation', 'source', 'support']
+ for subdirectory in subdirectories:
+ os.makedirs(os.path.join(ard_dir, subdirectory), exist_ok=True)
+
+ # prepare raster write options; https://gdal.org/drivers/raster/cog.html
+ write_options_base = ['BLOCKSIZE={}'.format(blocksize),
+ 'OVERVIEW_RESAMPLING={}'.format(ovr_resampling)]
+ write_options = dict()
+ for key in LERC_ERR_THRES:
+ write_options[key] = write_options_base.copy()
+ if compress is not None:
+ entry = 'COMPRESS={}'.format(compress)
+ write_options[key].append(entry)
+ if compress.startswith('LERC'):
+ entry = 'MAX_Z_ERROR={:f}'.format(LERC_ERR_THRES[key])
+ write_options[key].append(entry)
+
+ # create raster files: linear gamma0/sigma0 backscatter (-[vh|vv|hh|hv]-[gs]-lin.tif),
+ # ellipsoidal incident angle (-ei.tif), gamma-to-sigma ratio (-gs.tif),
+ # local contributing area (-lc.tif), local incident angle (-li.tif),
+ # noise power images (-np-[vh|vv|hh|hv].tif)
+ datasets_ard = dict()
+ for key in list(datasets_sar[0].keys()):
+ if key in ['dm', 'wm'] or key not in LERC_ERR_THRES.keys() or key not in allowed:
+ # raster files for keys 'dm' and 'wm' are created later
+ continue
+
+ meta_lower['suffix'] = key
+ outname_base = skeleton_files.format(**meta_lower)
+ if re.search('[gs]-lin', key):
+ subdir = 'measurement'
+ else:
+ subdir = 'annotation'
+ outname = os.path.join(ard_dir, subdir, outname_base)
+
+ if not os.path.isfile(outname):
+ print(outname)
+ images = [ds[key] for ds in datasets_sar]
+ ras = None
+ if len(images) > 1:
+ ras = Raster(images, list_separate=False)
+ source = ras.filename
+ else:
+ source = tempfile.NamedTemporaryFile(suffix='.vrt').name
+ gdalbuildvrt(src=images[0], dst=source)
+
+ # modify temporary VRT to make sure overview levels and resampling are properly applied
+ vrt_add_overviews(vrt=source, overviews=overviews, resampling=ovr_resampling)
+
+ options = {'format': driver, 'outputBounds': bounds,
+ 'dstNodata': dst_nodata_float, 'multithread': multithread,
+ 'creationOptions': write_options[key]}
+
+ gdalwarp(src=source, dst=outname, **options)
+ if ras is not None:
+ ras.close()
+ datasets_ard[key] = outname
+
+ # define a reference raster from the annotation datasets and list all gamma0/sigma0 backscatter measurement rasters
+ measure_tifs = [v for k, v in datasets_ard.items() if re.search('[gs]-lin', k)]
+ ref_key = list(datasets_ard.keys())[0]
+ ref_tif = datasets_ard[ref_key]
+
+ # create data mask raster (-dm.tif)
+ if 'dm' in allowed:
+ if wbm is not None:
+ if not config['dem_type'] == 'GETASSE30' and not os.path.isfile(wbm):
+ raise FileNotFoundError('External water body mask could not be found: {}'.format(wbm))
+
+ dm_path = ref_tif.replace(f'-{ref_key}.tif', '-dm.tif')
+ if not os.path.isfile(dm_path):
+ create_data_mask(outname=dm_path, datasets=datasets_sar, extent=extent, epsg=epsg,
+ driver=driver, creation_opt=write_options['dm'],
+ overviews=overviews, overview_resampling=ovr_resampling,
+ dst_nodata=dst_nodata_byte, wbm=wbm, product_type=product_type)
+ datasets_ard['dm'] = dm_path
+
+ # create acquisition ID image raster (-id.tif)
+ if 'id' in allowed:
+ id_path = ref_tif.replace(f'-{ref_key}.tif', '-id.tif')
+ if not os.path.isfile(id_path):
+ create_acq_id_image(outname=id_path, ref_tif=ref_tif,
+ datasets=datasets_sar, src_ids=src_ids,
+ extent=extent, epsg=epsg, driver=driver,
+ creation_opt=write_options['id'],
+ overviews=overviews, dst_nodata=dst_nodata_byte)
+ datasets_ard['id'] = id_path
+
+ # create DEM (-em.tif)
+ if dem_type is not None and 'em' in allowed:
+ if kml is None:
+ raise RuntimeError("If 'dem_type' is not None, `kml` needs to be defined.")
+ em_path = ref_tif.replace(f'-{ref_key}.tif', '-em.tif')
+ if not os.path.isfile(em_path):
+ print(em_path)
+ with Raster(ref_tif) as ras:
+ tr = ras.res
+ log_pyro = logging.getLogger('pyroSAR')
+ level = log_pyro.level
+ log_pyro.setLevel('NOTSET')
+ dem.to_mgrs(dem_type=dem_type, dst=em_path, kml=kml,
+ overviews=overviews, tile=tile, tr=tr,
+ create_options=write_options['em'],
+ pbar=False)
+ log_pyro.setLevel(level)
+ datasets_ard['em'] = em_path
+
+ # create color composite VRT (-cc-[gs]-lin.vrt)
+ if meta['polarization'] in ['DH', 'DV'] and len(measure_tifs) == 2:
+ cc_path = re.sub('[hv]{2}', 'cc', measure_tifs[0]).replace('.tif', '.vrt')
+ if not os.path.isfile(cc_path):
+ create_rgb_vrt(outname=cc_path, infiles=measure_tifs,
+ overviews=overviews, overview_resampling=ovr_resampling)
+ key = re.search('cc-[gs]-lin', cc_path).group()
+ datasets_ard[key] = cc_path
+
+ # create log-scaled gamma0|sigma0 nought VRTs (-[vh|vv|hh|hv]-[gs]-log.vrt)
+ fun = 'dB'
+ args = {'fact': 10}
+ scale = None
+ for item in measure_tifs:
+ target = item.replace('lin.tif', 'log.vrt')
+ if not os.path.isfile(target):
+ print(target)
+ create_vrt(src=item, dst=target, fun=fun, scale=scale,
+ args=args, options=vrt_options, overviews=overviews,
+ overview_resampling=ovr_resampling)
+ key = re.search('[hv]{2}-[gs]-log', target).group()
+ datasets_ard[key] = target
+
+ # create sigma nought RTC VRTs (-[vh|vv|hh|hv]-s-[lin|log].vrt)
+ if 'gs' in allowed:
+ gs_path = datasets_ard['gs']
+ for item in measure_tifs:
+ sigma0_rtc_lin = item.replace('g-lin.tif', 's-lin.vrt')
+ sigma0_rtc_log = item.replace('g-lin.tif', 's-log.vrt')
+
+ if not os.path.isfile(sigma0_rtc_lin):
+ print(sigma0_rtc_lin)
+ create_vrt(src=[item, gs_path], dst=sigma0_rtc_lin, fun='mul',
+ relpaths=True, options=vrt_options, overviews=overviews,
+ overview_resampling=ovr_resampling)
+ key = re.search('[hv]{2}-s-lin', sigma0_rtc_lin).group()
+ datasets_ard[key] = sigma0_rtc_lin
+
+ if not os.path.isfile(sigma0_rtc_log):
+ print(sigma0_rtc_log)
+ create_vrt(src=sigma0_rtc_lin, dst=sigma0_rtc_log, fun=fun,
+ scale=scale, options=vrt_options, overviews=overviews,
+ overview_resampling=ovr_resampling, args=args)
+ key = key.replace('lin', 'log')
+ datasets_ard[key] = sigma0_rtc_log
+
+ # create gamma nought RTC VRTs (-[vh|vv|hh|hv]-g-[lin|log].vrt)
+ if 'sg' in allowed:
+ sg_path = datasets_ard['sg']
+ for item in measure_tifs:
+ if not item.endswith('s-lin.tif'):
+ continue
+ gamma0_rtc_lin = item.replace('s-lin.tif', 'g-lin.vrt')
+ gamma0_rtc_log = item.replace('s-lin.tif', 'g-log.vrt')
+
+ if not os.path.isfile(gamma0_rtc_lin):
+ print(gamma0_rtc_lin)
+ create_vrt(src=[item, sg_path], dst=gamma0_rtc_lin, fun='mul',
+ relpaths=True, options=vrt_options, overviews=overviews,
+ overview_resampling=ovr_resampling)
+ key = re.search('[hv]{2}-g-lin', gamma0_rtc_lin).group()
+ datasets_ard[key] = gamma0_rtc_lin
+
+ if not os.path.isfile(gamma0_rtc_log):
+ print(gamma0_rtc_log)
+ create_vrt(src=gamma0_rtc_lin, dst=gamma0_rtc_log, fun=fun,
+ scale=scale, options=vrt_options, overviews=overviews,
+ overview_resampling=ovr_resampling, args=args)
+ key = key.replace('lin', 'log')
+ datasets_ard[key] = gamma0_rtc_log
+
+ # create backscatter wind model (-wm.tif)
+ # and wind normalization VRT (-[vv|hh]-s-lin-wn.vrt)
+ if 'wm' in annotation:
+ wm = []
+ for i, ds in enumerate(datasets_sar):
+ if 'wm' in ds.keys():
+ wm.append(ds['wm'])
+ else:
+ scene_base = os.path.basename(src_ids[i].scene)
+ raise RuntimeError(f'could not find wind model product for scene {scene_base}')
+
+ copol = 'VV' if 'VV' in src_ids[0].polarizations else 'HH'
+ copol_sigma0_key = f'{copol.lower()}-s-lin'
+ if copol_sigma0_key in datasets_ard.keys():
+ copol_sigma0 = datasets_ard[copol_sigma0_key]
+ wn_ard = re.sub(r's-lin\.(?:tif|vrt)', 's-lin-wn.vrt', copol_sigma0)
+ else:
+ copol_sigma0 = None
+ wn_ard = None
+
+ meta_lower['suffix'] = 'wm'
+ outname_base = skeleton_files.format(**meta_lower)
+ wm_ard = os.path.join(ard_dir, 'annotation', outname_base)
+
+ gapfill = True if src_ids[0].product == 'GRD' else False
+
+ wind_normalization(src=wm, dst_wm=wm_ard, dst_wn=wn_ard, measurement=copol_sigma0,
+ gapfill=gapfill, bounds=bounds, epsg=epsg, driver=driver, creation_opt=write_options['wm'],
+ dst_nodata=dst_nodata_float, multithread=multithread)
+ datasets_ard['wm'] = wm_ard
+ datasets_ard[f'{copol_sigma0_key}-wn'] = wn_ard
+
+ # copy support files
+ schema_dir = os.path.join(S1_NRB.__path__[0], 'validation', 'schemas')
+ schemas = os.listdir(schema_dir)
+ for schema in schemas:
+ schema_in = os.path.join(schema_dir, schema)
+ schema_out = os.path.join(ard_dir, 'support', schema)
+ if not os.path.isfile(schema_out):
+ print(schema_out)
+ shutil.copy(schema_in, schema_out)
+
+ # create metadata files in XML and (STAC) JSON formats
+ start = datetime.strptime(ard_start, '%Y%m%dT%H%M%S')
+ stop = datetime.strptime(ard_stop, '%Y%m%dT%H%M%S')
+ meta = extract.meta_dict(config=config, target=ard_dir, src_ids=src_ids, sar_dir=datadir,
+ proc_time=proc_time, start=start, stop=stop, compression=compress,
+ product_type=product_type)
+ ard_assets = sorted(sorted(list(datasets_ard.values()), key=lambda x: os.path.splitext(x)[1]),
+ key=lambda x: os.path.basename(os.path.dirname(x)), reverse=True)
+ if config['meta']['copy_original']:
+ copy_src_meta(ard_dir=ard_dir, src_ids=src_ids)
+ if 'OGC' in config['meta']['format']:
+ xml.parse(meta=meta, target=ard_dir, assets=ard_assets, exist_ok=True)
+ if 'STAC' in config['meta']['format']:
+ stac.parse(meta=meta, target=ard_dir, assets=ard_assets, exist_ok=True)
+ return str(round((time.time() - start_time), 2))
+
+
+def get_datasets(scenes, datadir, extent, epsg):
+ """
+ Collect processing output for a list of scenes.
+ Reads metadata from all source SLC/GRD scenes, finds matching output files in `datadir`
+ and filters both lists depending on the actual overlap of each SLC/GRD valid data coverage
+ with the current MGRS tile geometry. If no output is found for any scene the function will raise an error.
+ To obtain the extent of valid data coverage, first a binary
+ mask raster file is created with the name `datamask.tif`, which is stored in the same folder as
+ the processing output as found by :func:`~S1_NRB.snap.find_datasets`. Then, the boundary of this
+ binary mask is computed and stored as `datamask.gpkg` (see function :func:`spatialist.vector.boundary`).
+ If the provided `extent` does not overlap with this boundary, the output is discarded. This scenario
+ might occur when the scene's geometry read from its metadata overlaps with the tile but the actual
+ extent of data does not.
+
+ Parameters
+ ----------
+ scenes: list[str]
+ List of scenes to process. Either an individual scene or multiple, matching scenes (consecutive acquisitions).
+ datadir: str
+ The directory containing the SAR datasets processed from the source scenes using pyroSAR.
+ The function will raise an error if the processing output cannot be found for all scenes in `datadir`.
+ extent: dict
+ Spatial extent of the MGRS tile, derived from a :class:`~spatialist.vector.Vector` object.
+ epsg: int
+ The coordinate reference system as an EPSG code.
+
+ Returns
+ -------
+ ids: list[:class:`pyroSAR.drivers.ID`]
+ List of :class:`~pyroSAR.drivers.ID` objects of all source SLC/GRD scenes that overlap with the current MGRS tile.
+ datasets: list[dict]
+ List of SAR processing output files that match each :class:`~pyroSAR.drivers.ID` object of `ids`.
+ The format is a list of dictionaries per scene with keys as described by e.g. :func:`S1_NRB.snap.find_datasets`.
+
+ See Also
+ --------
+ :func:`S1_NRB.snap.find_datasets`
+ """
+ ids = identify_many(scenes)
+ datasets = []
+ for i, _id in enumerate(ids):
+ files = find_datasets(scene=_id.scene, outdir=datadir, epsg=epsg)
+ if files is not None:
+
+ base = os.path.splitext(os.path.basename(_id.scene))[0]
+ ocn = re.sub('(?:SLC_|GRD[FHM])_1', 'OCN__2', base)[:-5]
+ # allow 1 second tolerance
+ s_start = int(ocn[31])
+ s_stop = int(ocn[47])
+ ocn_list = list(ocn)
+ s = 1
+ ocn_list[31] = f'[{s_start - s}{s_start}{s_start + s}]'
+ ocn_list[47] = f'[{s_stop - s}{s_stop}{s_stop + s}]'
+ ocn = ''.join(ocn_list)
+ ocn_match = finder(target=datadir, matchlist=[ocn], regex=True, foldermode=2)
+ if len(ocn_match) > 0:
+ cmod = os.path.join(ocn_match[0], 'owiNrcsCmod.tif')
+ if os.path.isfile(cmod):
+ files['wm'] = cmod
+
+ datasets.append(files)
+ else:
+ base = os.path.basename(_id.scene)
+ raise RuntimeError(f'cannot find processing output for scene {base}')
+
+ i = 0
+ while i < len(datasets):
+ measurements = [datasets[i][x] for x in datasets[i].keys() if re.search('[gs]-lin', x)]
+ dm_ras = os.path.join(os.path.dirname(measurements[0]), 'datamask.tif')
+ dm_vec = dm_ras.replace('.tif', '.gpkg')
+
+ if not os.path.isfile(dm_ras):
+ with Raster(measurements[0]) as ras:
+ arr = ras.array()
+ mask = ~np.isnan(arr)
+ del arr
+ # remove scene if file does not contain valid data
+ if len(mask[mask == 1]) == 0:
+ del ids[i], datasets[i]
+ continue
+ with vectorize(target=mask, reference=ras) as vec:
+ with boundary(vec, expression="value=1") as bounds:
+ if not os.path.isfile(dm_ras):
+ rasterize(vectorobject=bounds, reference=ras, outname=dm_ras)
+ if not os.path.isfile(dm_vec):
+ bounds.write(outfile=dm_vec)
+ del mask
+ if not os.path.isfile(dm_vec):
+ with Raster(dm_ras) as ras:
+ mask = ras.array().astype('bool')
+ # remove scene if file does not contain valid data
+ if len(mask[mask == 1]) == 0:
+ del ids[i], datasets[i]
+ continue
+ with vectorize(target=mask, reference=ras) as vec:
+ boundary(vec, expression="value=1", outname=dm_vec)
+ del mask
+ with Vector(dm_vec) as bounds:
+ with bbox(extent, epsg) as tile_geom:
+ inter = intersect(bounds, tile_geom)
+ if inter is None:
+ del ids[i]
+ del datasets[i]
+ else:
+ # Add dm_ras to the datasets if it overlaps with the current tile
+ datasets[i]['datamask'] = dm_ras
+ i += 1
+ inter.close()
+ return ids, datasets
+
+
+def create_vrt(src, dst, fun, relpaths=False, scale=None, offset=None, args=None,
+ options=None, overviews=None, overview_resampling=None):
+ """
+ Creates a VRT file for the specified source dataset(s) and adds a pixel function that should be applied on the fly
+ when opening the VRT file.
+
+ Parameters
+ ----------
+ src: str or list[str]
+ The input dataset(s).
+ dst: str
+ The output dataset.
+ fun: str
+ A `PixelFunctionType` that should be applied on the fly when opening the VRT file. The function is applied to a
+ band that derives its pixel information from the source bands. A list of possible options can be found here:
+ https://gdal.org/drivers/raster/vrt.html#default-pixel-functions.
+ Furthermore, the option 'decibel' can be specified, which will implement a custom pixel function that uses
+ Python code for decibel conversion (10*log10).
+ relpaths: bool
+ Should all `SourceFilename` XML elements with attribute `@relativeToVRT="0"` be updated to be paths relative to
+ the output VRT file? Default is False.
+ scale: int or None
+ The scale that should be applied when computing “real” pixel values from scaled pixel values on a raster band.
+ Will be ignored if `fun='decibel'`.
+ offset: float or None
+ The offset that should be applied when computing “real” pixel values from scaled pixel values on a raster band.
+ Will be ignored if `fun='decibel'`.
+ args: dict or None
+ arguments for `fun` passed as `PixelFunctionArguments`. Requires GDAL>=3.5 to be read.
+ options: dict or None
+ Additional parameters passed to `gdal.BuildVRT`.
+ overviews: list[int] or None
+ Internal overview levels to be created for each raster file.
+ overview_resampling: str or None
+ Resampling method for overview levels.
+
+ Examples
+ --------
+ linear gamma0 backscatter as input:
+
+ >>> src = 's1a-iw-nrb-20220601t052704-043465-0530a1-32tpt-vh-g-lin.tif'
+
+ decibel scaling I:
+ use `log10` pixel function and additional `Scale` parameter.
+ Known to display well in QGIS, but `Scale` is ignored when reading array in Python.
+
+ >>> dst = src.replace('-lin.tif', '-log1.vrt')
+ >>> create_vrt(src=src, dst=dst, fun='log10', scale=10)
+
+ decibel scaling II:
+ use custom Python pixel function. Requires additional environment variable GDAL_VRT_ENABLE_PYTHON set to YES.
+
+ >>> dst = src.replace('-lin.tif', '-log2.vrt')
+ >>> create_vrt(src=src, dst=dst, fun='decibel')
+
+ decibel scaling III:
+ use `dB` pixel function with additional `PixelFunctionArguments`. Works best but requires GDAL>=3.5.
+
+ >>> dst = src.replace('-lin.tif', '-log3.vrt')
+ >>> create_vrt(src=src, dst=dst, fun='dB', args={'fact': 10})
+ """
+ options = {} if options is None else options
+ gdalbuildvrt(src=src, dst=dst, **options)
+ tree = etree.parse(dst)
+ root = tree.getroot()
+ band = tree.find('VRTRasterBand')
+ band.attrib['subClass'] = 'VRTDerivedRasterBand'
+
+ if fun == 'decibel':
+ pxfun_language = etree.SubElement(band, 'PixelFunctionLanguage')
+ pxfun_language.text = 'Python'
+ pxfun_type = etree.SubElement(band, 'PixelFunctionType')
+ pxfun_type.text = fun
+ pxfun_code = etree.SubElement(band, 'PixelFunctionCode')
+ pxfun_code.text = etree.CDATA("""
+ import numpy as np
+ def decibel(in_ar, out_ar, xoff, yoff, xsize, ysize, raster_xsize, raster_ysize, buf_radius, gt, **kwargs):
+ np.multiply(np.log10(in_ar[0], where=in_ar[0]>0.0, out=out_ar, dtype='float32'), 10.0, out=out_ar, dtype='float32')
+ """)
+ else:
+ pixfun_type = etree.SubElement(band, 'PixelFunctionType')
+ pixfun_type.text = fun
+ if args is not None:
+ arg = etree.SubElement(band, 'PixelFunctionArguments')
+ for key, value in args.items():
+ arg.attrib[key] = str(value)
+ if scale is not None:
+ sc = etree.SubElement(band, 'Scale')
+ sc.text = str(scale)
+ if offset is not None:
+ off = etree.SubElement(band, 'Offset')
+ off.text = str(offset)
+
+ if any([overviews, overview_resampling]) is not None:
+ ovr = tree.find('OverviewList')
+ if ovr is None:
+ ovr = etree.SubElement(root, 'OverviewList')
+ if overview_resampling is not None:
+ ovr.attrib['resampling'] = overview_resampling.lower()
+ if overviews is not None:
+ ov = str(overviews)
+ for x in ['[', ']', ',']:
+ ov = ov.replace(x, '')
+ ovr.text = ov
+
+ if relpaths:
+ srcfiles = tree.xpath('//SourceFilename[@relativeToVRT="0"]')
+ for srcfile in srcfiles:
+ repl = os.path.relpath(srcfile.text, start=os.path.dirname(dst))
+ repl = repl.replace('\\', '/')
+ srcfile.text = repl
+ srcfile.attrib['relativeToVRT'] = '1'
+
+ etree.indent(root)
+ tree.write(dst, pretty_print=True, xml_declaration=False, encoding='utf-8')
+
+
+def create_rgb_vrt(outname, infiles, overviews, overview_resampling):
+ """
+ Creation of the color composite VRT file.
+
+ Parameters
+ ----------
+ outname: str
+ Full path to the output VRT file.
+ infiles: list[str]
+ A list of paths pointing to the linear scaled measurement backscatter files.
+ overviews: list[int]
+ Internal overview levels to be defined for the created VRT file.
+ overview_resampling: str
+ Resampling method applied to overview pyramids.
+ """
+ print(outname)
+
+ # make sure order is right and co-polarization (VV or HH) is first
+ pols = [re.search('[hv]{2}', os.path.basename(f)).group() for f in infiles]
+ if pols[1] in ['vv', 'hh']:
+ infiles.reverse()
+ pols.reverse()
+
+ # format overview levels
+ ov = str(overviews)
+ for x in ['[', ']', ',']:
+ ov = ov.replace(x, '')
+
+ # create VRT file and change its content
+ gdalbuildvrt(src=infiles, dst=outname, separate=True)
+
+ tree = etree.parse(outname)
+ root = tree.getroot()
+ srs = tree.find('SRS').text
+ geotrans = tree.find('GeoTransform').text
+ bands = tree.findall('VRTRasterBand')
+ vrt_nodata = bands[0].find('NoDataValue').text
+ complex_src = [band.find('ComplexSource') for band in bands]
+ for cs in complex_src:
+ cs.remove(cs.find('NODATA'))
+
+ new_band = etree.SubElement(root, 'VRTRasterBand',
+ attrib={'dataType': 'Float32', 'band': '3',
+ 'subClass': 'VRTDerivedRasterBand'})
+ new_band_na = etree.SubElement(new_band, 'NoDataValue')
+ new_band_na.text = 'nan'
+ pxfun_type = etree.SubElement(new_band, 'PixelFunctionType')
+ pxfun_type.text = 'mul'
+ for cs in complex_src:
+ new_band.append(deepcopy(cs))
+
+ src = new_band.findall('ComplexSource')[1]
+ fname = src.find('SourceFilename')
+ fname_old = fname.text
+ src_attr = src.find('SourceProperties').attrib
+ fname.text = etree.CDATA("""
+
+ {srs}
+ {geotrans}
+
+ {vrt_nodata}
+ {px_fun}
+
+ {fname}
+ 1
+
+
+
+
+
+ {ov}
+
+ """.format(rasterxsize=src_attr['RasterXSize'], rasterysize=src_attr['RasterYSize'], srs=srs, geotrans=geotrans,
+ dtype=src_attr['DataType'], px_fun='inv', fname=fname_old, vrt_nodata=vrt_nodata,
+ blockxsize=src_attr['BlockXSize'], blockysize=src_attr['BlockYSize'],
+ ov_resampling=overview_resampling.lower(), ov=ov))
+
+ bands = tree.findall('VRTRasterBand')
+ for band, col in zip(bands, ['Red', 'Green', 'Blue']):
+ color = etree.Element('ColorInterp')
+ color.text = col
+ band.insert(0, color)
+
+ ovr = etree.SubElement(root, 'OverviewList', attrib={'resampling': overview_resampling.lower()})
+ ovr.text = ov
+
+ etree.indent(root)
+ tree.write(outname, pretty_print=True, xml_declaration=False, encoding='utf-8')
+
+
+def calc_product_start_stop(src_ids, extent, epsg):
+ """
+ Calculates the start and stop times of the ARD product.
+ The geolocation grid points including their azimuth time information are extracted first from the metadata of each
+ source SLC. These grid points are then used to interpolate the azimuth time for the lower right and upper left
+ (ascending) or upper right and lower left (descending) corners of the MGRS tile extent.
+
+ Parameters
+ ----------
+ src_ids: list[pyroSAR.drivers.ID]
+ List of :class:`~pyroSAR.drivers.ID` objects of all source SLC scenes that overlap with the current MGRS tile.
+ extent: dict
+ Spatial extent of the MGRS tile, derived from a :class:`~spatialist.vector.Vector` object.
+ epsg: int
+ The coordinate reference system as an EPSG code.
+
+ Returns
+ -------
+ tuple[str]
+
+ - Start time of the ARD product formatted as `%Y%m%dT%H%M%S` in UTC.
+ - Stop time of the ARD product formatted as `%Y%m%dT%H%M%S` in UTC.
+ """
+ with bbox(extent, epsg) as tile_geom:
+ tile_geom.reproject(4326)
+ ext = tile_geom.extent
+ ul = (ext['xmin'], ext['ymax'])
+ ur = (ext['xmax'], ext['ymax'])
+ lr = (ext['xmax'], ext['ymin'])
+ ll = (ext['xmin'], ext['ymin'])
+ tile_geom = None
+
+ slc_dict = {}
+ for i, sid in enumerate(src_ids):
+ uid = os.path.basename(sid.scene).split('.')[0][-4:]
+ slc_dict[uid] = get_src_meta(sid)
+ slc_dict[uid]['sid'] = sid
+
+ uids = list(slc_dict.keys())
+
+ for uid in uids:
+ t = find_in_annotation(annotation_dict=slc_dict[uid]['annotation'],
+ pattern='.//geolocationGridPoint/azimuthTime')
+ swaths = t.keys()
+ y = find_in_annotation(annotation_dict=slc_dict[uid]['annotation'], pattern='.//geolocationGridPoint/latitude')
+ x = find_in_annotation(annotation_dict=slc_dict[uid]['annotation'], pattern='.//geolocationGridPoint/longitude')
+ t_flat = np.asarray([datetime.fromisoformat(item).timestamp() for sublist in [t[swath] for swath in swaths]
+ for item in sublist])
+ y_flat = np.asarray([float(item) for sublist in [y[swath] for swath in swaths] for item in sublist])
+ x_flat = np.asarray([float(item) for sublist in [x[swath] for swath in swaths] for item in sublist])
+ g = np.asarray([(x, y) for x, y in zip(x_flat, y_flat)])
+ slc_dict[uid]['az_time'] = t_flat
+ slc_dict[uid]['gridpts'] = g
+
+ if len(uids) == 2:
+ starts = [datetime.strptime(slc_dict[key]['sid'].start, '%Y%m%dT%H%M%S') for key in slc_dict.keys()]
+ if starts[0] > starts[1]:
+ az_time = np.concatenate([slc_dict[uids[1]]['az_time'], slc_dict[uids[0]]['az_time']])
+ gridpts = np.concatenate([slc_dict[uids[1]]['gridpts'], slc_dict[uids[0]]['gridpts']])
+ else:
+ az_time = np.concatenate([slc_dict[key]['az_time'] for key in slc_dict.keys()])
+ gridpts = np.concatenate([slc_dict[key]['gridpts'] for key in slc_dict.keys()])
+ else:
+ az_time = slc_dict[uids[0]]['az_time']
+ gridpts = slc_dict[uids[0]]['gridpts']
+
+ if slc_dict[uids[0]]['sid'].orbit == 'A':
+ coord1 = lr
+ coord2 = ul
+ else:
+ coord1 = ur
+ coord2 = ll
+
+ method = 'linear'
+ res = [griddata(gridpts, az_time, coord1, method=method),
+ griddata(gridpts, az_time, coord2, method=method)]
+
+ min_start = min([datetime.strptime(slc_dict[uid]['sid'].start, '%Y%m%dT%H%M%S') for uid in uids])
+ max_stop = max([datetime.strptime(slc_dict[uid]['sid'].stop, '%Y%m%dT%H%M%S') for uid in uids])
+ res_t = []
+ for i, r in enumerate(res):
+ if np.isnan(r):
+ if i == 0:
+ res_t.append(min_start)
+ else:
+ res_t.append(max_stop)
+ else:
+ res_t.append(datetime.fromtimestamp(float(r)))
+
+ start = datetime.strftime(res_t[0], '%Y%m%dT%H%M%S')
+ stop = datetime.strftime(res_t[1], '%Y%m%dT%H%M%S')
+
+ return start, stop
+
+
+def create_data_mask(outname, datasets, extent, epsg, driver, creation_opt,
+ overviews, overview_resampling, dst_nodata, product_type, wbm=None):
+ """
+ Creation of the Data Mask image.
+
+ Parameters
+ ----------
+ outname: str
+ Full path to the output data mask file.
+ datasets: list[dict]
+ List of processed output files that match the source scenes and overlap with the current MGRS tile.
+ An error will be thrown if not all datasets contain a key `datamask`.
+ The function will return without an error if not all datasets contain a key `dm`.
+ extent: dict
+ Spatial extent of the MGRS tile, derived from a :class:`~spatialist.vector.Vector` object.
+ epsg: int
+ The coordinate reference system as an EPSG code.
+ driver: str
+ GDAL driver to use for raster file creation.
+ creation_opt: list[str]
+ GDAL creation options to use for raster file creation. Should match specified GDAL driver.
+ overviews: list[int]
+ Internal overview levels to be created for each raster file.
+ overview_resampling: str
+ Resampling method for overview levels.
+ dst_nodata: int or str
+ Nodata value to write to the output raster.
+ product_type: str
+ The type of ARD product that is being created. Either 'NRB' or 'ORB'.
+ wbm: str or None
+ Path to a water body mask file with the dimensions of an MGRS tile. Optional if `product_type='NRB', mandatory
+ if `product_type='ORB'`.
+ """
+ measurement_keys = [x for x in datasets[0].keys() if re.search('[gs]-lin', x)]
+ measurement = [scene[measurement_keys[0]] for scene in datasets]
+ datamask = [scene['datamask'] for scene in datasets]
+ ls = []
+ for scene in datasets:
+ if 'dm' in scene:
+ ls.append(scene['dm'])
+ else:
+ return # do not create a data mask if not all scenes have a layover-shadow mask
+ print(outname)
+ if product_type == 'NRB':
+ dm_bands = [{'arr_val': 0, 'name': 'not layover, nor shadow'},
+ {'arr_val': 1, 'name': 'layover'},
+ {'arr_val': 2, 'name': 'shadow'},
+ # {'arr_val': 3, 'name': 'layover and shadow'}, # just for context, not used as an individual band
+ {'arr_val': 4, 'name': 'ocean water'}]
+ elif product_type == 'ORB':
+ if wbm is None:
+ raise RuntimeError('Water body mask is required for ORB products')
+ dm_bands = [{'arr_val': [0, 1, 2], 'name': 'land'},
+ {'arr_val': 4, 'name': 'ocean water'}]
+ else:
+ raise RuntimeError(f'Unknown product type: {product_type}')
+
+ tile_bounds = [extent['xmin'], extent['ymin'], extent['xmax'], extent['ymax']]
+
+ vrt_ls = '/vsimem/' + os.path.dirname(outname) + 'ls.vrt'
+ vrt_valid = '/vsimem/' + os.path.dirname(outname) + 'valid.vrt'
+ vrt_measurement = '/vsimem/' + os.path.dirname(outname) + 'measurement.vrt'
+ gdalbuildvrt(src=ls, dst=vrt_ls, outputBounds=tile_bounds, void=False)
+ gdalbuildvrt(src=datamask, dst=vrt_valid, outputBounds=tile_bounds, void=False)
+ gdalbuildvrt(src=measurement, dst=vrt_measurement, outputBounds=tile_bounds, void=False)
+
+ with Raster(vrt_ls) as ras_ls:
+ with bbox(extent, crs=epsg) as tile_vec:
+ ras_ls_res = ras_ls.res
+ rows = ras_ls.rows
+ cols = ras_ls.cols
+ geotrans = ras_ls.raster.GetGeoTransform()
+ proj = ras_ls.raster.GetProjection()
+ arr_dm = ras_ls.array()
+
+ # Add Water Body Mask
+ if wbm is not None:
+ with Raster(wbm) as ras_wbm:
+ ras_wbm_cols = ras_wbm.cols
+ cols_ratio = ras_wbm_cols / cols
+ if cols_ratio > 1:
+ res = int(ras_ls_res[0])
+ wbm_lowres = wbm.replace('.tif', f'_{res}m.vrt')
+ options = {'xRes': res, 'yRes': res,
+ 'resampleAlg': 'mode'}
+ gdalbuildvrt(src=wbm, dst=wbm_lowres, **options)
+ with Raster(wbm_lowres) as ras_wbm_lowres:
+ arr_wbm = ras_wbm_lowres.array()
+ else:
+ arr_wbm = ras_wbm.array()
+ out_arr = np.where((arr_wbm == 1), 4, arr_dm)
+ del arr_wbm
+ else:
+ out_arr = arr_dm
+ del dm_bands[3]
+ del arr_dm
+
+ # Extend the shadow class of the data mask with nodata values from backscatter data and create final array
+ with Raster(vrt_valid)[tile_vec] as ras_valid:
+ with Raster(vrt_measurement)[tile_vec] as ras_measurement:
+ arr_valid = ras_valid.array()
+ arr_measurement = ras_measurement.array()
+
+ out_arr = np.nan_to_num(out_arr)
+ out_arr = np.where(((arr_valid == 1) & (np.isnan(arr_measurement)) & (out_arr != 4)), 2,
+ out_arr)
+ out_arr[np.isnan(arr_valid)] = dst_nodata
+ del arr_measurement
+ del arr_valid
+
+ outname_tmp = '/vsimem/' + os.path.basename(outname) + '.vrt'
+ gdriver = gdal.GetDriverByName('GTiff')
+ ds_tmp = gdriver.Create(outname_tmp, rows, cols, len(dm_bands), gdal.GDT_Byte,
+ options=['ALPHA=UNSPECIFIED', 'PHOTOMETRIC=MINISWHITE'])
+ gdriver = None
+ ds_tmp.SetGeoTransform(geotrans)
+ ds_tmp.SetProjection(proj)
+
+ for i, _dict in enumerate(dm_bands):
+ band = ds_tmp.GetRasterBand(i + 1)
+ arr_val = _dict['arr_val']
+ b_name = _dict['name']
+
+ arr = np.full((rows, cols), 0)
+ arr[out_arr == dst_nodata] = dst_nodata
+ if arr_val in [0, 4]:
+ arr[out_arr == arr_val] = 1
+ elif arr_val in [1, 2]:
+ arr[(out_arr == arr_val) | (out_arr == 3)] = 1
+ elif arr_val == [0, 1, 2]:
+ arr[out_arr != 4] = 1
+
+ arr = arr.astype('uint8')
+ band.WriteArray(arr)
+ band.SetNoDataValue(dst_nodata)
+ band.SetDescription(b_name)
+ band.FlushCache()
+ band = None
+ del arr
+
+ ds_tmp.SetMetadataItem('TIFFTAG_DATETIME', strftime('%Y:%m:%d %H:%M:%S', gmtime()))
+ ds_tmp.BuildOverviews(overview_resampling, overviews)
+ outDataset_cog = gdal.GetDriverByName(driver).CreateCopy(outname, ds_tmp, strict=1, options=creation_opt)
+ outDataset_cog = None
+ ds_tmp = None
+ tile_vec = None
+
+
+def create_acq_id_image(outname, ref_tif, datasets, src_ids, extent,
+ epsg, driver, creation_opt, overviews, dst_nodata):
+ """
+ Creation of the Acquisition ID image.
+
+ Parameters
+ ----------
+ outname: str
+ Full path to the output data mask file.
+ ref_tif: str
+ Full path to any GeoTIFF file of the ARD product.
+ datasets: list[dict]
+ List of processed output files that match the source SLC scenes and overlap with the current MGRS tile.
+ src_ids: list[pyroSAR.drivers.ID]
+ List of :class:`~pyroSAR.drivers.ID` objects of all source SLC scenes that overlap with the current MGRS tile.
+ extent: dict
+ Spatial extent of the MGRS tile, derived from a :class:`~spatialist.vector.Vector` object.
+ epsg: int
+ The CRS used for the ARD product; provided as an EPSG code.
+ driver: str
+ GDAL driver to use for raster file creation.
+ creation_opt: list[str]
+ GDAL creation options to use for raster file creation. Should match specified GDAL driver.
+ overviews: list[int]
+ Internal overview levels to be created for each raster file.
+ dst_nodata: int or str
+ Nodata value to write to the output raster.
+ """
+ print(outname)
+ src_scenes = [sid.scene for sid in src_ids]
+ # If there are two source scenes, make sure that the order of acquisitions in all lists is correct!
+ if len(src_scenes) > 1:
+ if not len(src_scenes) == 2 and len(datasets) == 2:
+ raise RuntimeError('expected lists `src_scenes` and `valid_mask_list` to be of length 2; length is '
+ '{} and {} respectively'.format(len(src_scenes), len(datasets)))
+ starts_src = [datetime.strptime(identify(f).start, '%Y%m%dT%H%M%S') for f in src_scenes]
+ start_valid = [re.search('[0-9]{8}T[0-9]{6}', os.path.basename(x)).group() for x in src_scenes]
+ start_valid = [datetime.strptime(x, '%Y%m%dT%H%M%S') for x in start_valid]
+ if starts_src[0] > starts_src[1]:
+ src_scenes.reverse()
+ starts_src.reverse()
+ if start_valid[0] != starts_src[0]:
+ datasets.reverse()
+ if start_valid[0] != starts_src[0]:
+ raise RuntimeError('failed to match order of lists `src_scenes` and `valid_mask_list`')
+
+ tile_bounds = [extent['xmin'], extent['ymin'], extent['xmax'], extent['ymax']]
+
+ arr_list = []
+ for dataset in datasets:
+ vrt_valid = '/vsimem/' + os.path.dirname(outname) + 'mosaic.vrt'
+ gdalbuildvrt(src=dataset['datamask'], dst=vrt_valid, outputBounds=tile_bounds, void=False)
+ with bbox(extent, crs=epsg) as tile_vec:
+ with Raster(vrt_valid)[tile_vec] as vrt_ras:
+ vrt_arr = vrt_ras.array()
+ arr_list.append(vrt_arr)
+ del vrt_arr
+ tile_vec = None
+
+ src_scenes_clean = [os.path.basename(src).replace('.zip', '').replace('.SAFE', '') for src in src_scenes]
+ tag = '{{"{src1}": 1}}'.format(src1=src_scenes_clean[0])
+ out_arr = np.full(arr_list[0].shape, dst_nodata)
+ out_arr[arr_list[0] == 1] = 1
+ if len(arr_list) == 2:
+ out_arr[arr_list[1] == 1] = 2
+ tag = '{{"{src1}": 1, "{src2}": 2}}'.format(src1=src_scenes_clean[0], src2=src_scenes_clean[1])
+
+ creation_opt.append('TIFFTAG_IMAGEDESCRIPTION={}'.format(tag))
+ with Raster(ref_tif) as ref_ras:
+ ref_ras.write(outname, format=driver, array=out_arr.astype('uint8'), nodata=dst_nodata, overwrite=True,
+ overviews=overviews, options=creation_opt)
+
+
+def wind_normalization(src, dst_wm, dst_wn, measurement, gapfill, bounds, epsg, driver, creation_opt, dst_nodata,
+ multithread, resolution=915):
+ """
+ Create wind normalization layers. A wind model annotation layer is created and optionally
+ a wind normalization VRT.
+
+ Parameters
+ ----------
+ src: list[str]
+ A list of OCN products as prepared by :func:`S1_NRB.ocn.extract`
+ dst_wm: str
+ The name of the wind model layer in the ARD product
+ dst_wn: str or None
+ The name of the wind normalization VRT. If None, no VRT will be created.
+ Requires `measurement` to point to a file.
+ measurement: str or None
+ The name of the measurement file used for wind normalization in `dst_wn`.
+ If None, no wind normalization VRT will be created.
+ gapfill: bool
+ Perform additional gap filling (:func:`S1_NRB.ocn.gapfill`)?
+ This is recommended if the Level-1 source product of `measurement` is GRD
+ in which case gaps are introduced between subsequently acquired scenes.
+ bounds: list[float]
+ the bounds of the MGRS tile
+ epsg: int
+ The EPSG code of the MGRS tile
+ driver: str
+ GDAL driver to use for raster file creation.
+ creation_opt: list[str]
+ GDAL creation options to use for raster file creation. Should match specified GDAL driver.
+ dst_nodata: float
+ Nodata value to write to the output raster.
+ multithread: bool
+ Should `gdalwarp` use multithreading?
+ resolution: int, optional
+ The target pixel resolution in meters. 915 is chosen as default because it is closest
+ to the OCN product resolution (1000) and still fits into the MGRS bounds
+ (``109800 % 915 == 0``).
+
+ Returns
+ -------
+
+ """
+ if len(src) > 1:
+ cmod_mosaic = tempfile.NamedTemporaryFile(suffix='.tif').name
+ gdalwarp(src=src, dst=cmod_mosaic)
+ if gapfill:
+ cmod_geo = tempfile.NamedTemporaryFile(suffix='.tif').name
+ ocn.gapfill(src=cmod_mosaic, dst=cmod_geo, md=1, si=1)
+ else:
+ cmod_geo = cmod_mosaic
+ else:
+ cmod_geo = src[0]
+
+ if not os.path.isfile(dst_wm):
+ print(dst_wm)
+ gdalwarp(src=cmod_geo,
+ dst=dst_wm,
+ outputBounds=bounds,
+ dstSRS=f'EPSG:{epsg}',
+ xRes=resolution, yRes=resolution,
+ resampleAlg='bilinear',
+ format=driver,
+ dstNodata=dst_nodata,
+ multithread=multithread,
+ creationOptions=creation_opt)
+
+ if dst_wn is not None and measurement is not None:
+ if not os.path.isfile(dst_wn):
+ print(dst_wn)
+ with Raster(measurement) as ras:
+ xres, yres = ras.res
+ create_vrt(src=[measurement, dst_wm], dst=dst_wn, fun='div',
+ options={'xRes': xres, 'yRes': yres})
diff --git a/S1_NRB/metadata/extract.py b/S1_NRB/metadata/extract.py
index 589e18e..24a938f 100644
--- a/S1_NRB/metadata/extract.py
+++ b/S1_NRB/metadata/extract.py
@@ -593,10 +593,7 @@ def calc_enl(tif, block_size=30, return_arr=False, decimals=2):
References
----------
- .. [1] S. N. Anfinsen, A. P. Doulgeris and T. Eltoft,
- "Estimation of the Equivalent Number of Looks in Polarimetric Synthetic Aperture Radar Imagery,"
- in IEEE Transactions on Geoscience and Remote Sensing, vol. 47, no. 11, pp. 3795-3809, Nov. 2009,
- doi: 10.1109/TGRS.2009.2019269.
+ :cite:`anfinsen.etal_2009`
"""
with Raster(tif) as ras:
arr = ras.array()
diff --git a/docs/about/changelog.rst b/docs/about/changelog.rst
index 62ac1d6..8ba0322 100644
--- a/docs/about/changelog.rst
+++ b/docs/about/changelog.rst
@@ -57,7 +57,7 @@ Changelog
* New metadata config parameters (`#110 `_)
* support for scenes acquired in NRT Slicing mode (`#112 `_)
-`Full Changelog `_
+`Full v1.4.0 Changelog `_
1.3.0 | 2023-05-24
------------------
@@ -70,7 +70,7 @@ Changelog
* enhanced time filtering (`#84 `_)
* general processor improvements (`#85 `_)
-`Full Changelog `_
+`Full v1.3.0 Changelog `_
1.2.0 | 2022-12-29
------------------
@@ -81,7 +81,7 @@ Changelog
* add DEM as additional output layer (`#70 `_)
* sigma0 processing and annotation layer configuration (`#74 `_)
-`Full Changelog `_
+`Full v1.2.0 Changelog `_
1.1.0 | 2022-09-29
------------------
@@ -93,7 +93,7 @@ Changelog
* documentation update reflecting the recent process restructuring (`#66 `_)
* renamed processing mode 'snap' to 'rtc' (`#67 `_)
-`Full Changelog `_
+`Full v1.1.0 Changelog `_
1.0.2 | 2022-08-24
------------------
@@ -106,14 +106,14 @@ Changelog
* Fix and improve metadata geometry handling (`#57 `_)
* SNAP 9 compatibility (`#58 `_)
-`Full Changelog `_
+`Full v1.0.2 Changelog `_
1.0.1 | 2022-07-03
------------------
* dem handling improvements (`#45 `_)
-`Full Changelog `_
+`Full v1.0.1 Changelog `_
1.0.0 | 2022-06-23
------------------
@@ -125,7 +125,7 @@ Changelog
* Geolocation accuracy (`#40 `_)
* various bug fixes and documentation improvements
-`Full Changelog `_
+`Full v1.0.0 Changelog `_
0.4.2 | 2022-06-16
------------------
@@ -133,7 +133,7 @@ Changelog
* Update documentation (`#27 `_)
* find unpacked .SAFE scenes in scene_dir (instead of just .zip) (`aea53a5 `_)
-`Full Changelog `_
+`Full v0.4.2 Changelog `_
0.4.1 | 2022-06-01
------------------
@@ -142,7 +142,7 @@ Changelog
* set dem download authentication via env. variables (`#26 `_)
* various bug fixes
-`Full Changelog `_
+`Full v0.4.1 Changelog `_
0.4.0 | 2022-05-30
------------------
@@ -156,7 +156,7 @@ Changelog
* Set up sphinx documentation (`#23 `_)
* AOI scene selection (`#24 `_)
-`Full Changelog `_
+`Full v0.4.0 Changelog `_
0.3.0 | 2022-03-30
------------------
@@ -170,13 +170,9 @@ Changelog
* Improved RGB composite (`#15 `_)
* Store DEM/WBM tiles in UTM zones different to the native MGRS zone (`#16 `_)
-`Full Changelog `_
+`Full v0.3.0 Changelog `_
0.2.0 | 2022-03-03
------------------
-`Full Changelog `_
-
-0.1.0 | 2022-01-14
-------------------
-
+`Full v0.2.0 Changelog `_
diff --git a/docs/general/enl.rst b/docs/general/enl.rst
index a35fd2d..2bacb0d 100644
--- a/docs/general/enl.rst
+++ b/docs/general/enl.rst
@@ -9,9 +9,9 @@ In case of linear scaled backscatter data, ENL can be calculated as:
.. math::
ENL = \frac{\mu^2}{\sigma^2}
-where :math:`\mu` is the mean and :math:`\sigma` is the standard deviation of the image. (:footcite:`cls_2016`, section A1.1.7)
+where :math:`\mu` is the mean and :math:`\sigma` is the standard deviation of the image. (:cite:`cls_2016`, section A1.1.7)
-The ENL value stored in the metadata of each S1-NRB product is calculated as suggested in :footcite:`anfinsen.etal_2009`, where ENL is first
+The ENL value stored in the metadata of each S1-NRB product is calculated as suggested in :cite:`anfinsen.etal_2009`, where ENL is first
calculated for small pixel windows over the cross-polarized backscatter image and afterwards the median value of
the distribution is selected.
@@ -40,7 +40,7 @@ the resulting array is shown in Figure 1.
Comparison between GRDH and NRB
-------------------------------
-:footcite:`cls_2016` provides estimates of ENL for different Sentinel-1 products (average over all swaths), e.g. ENL of 4.4 for GRDH in
+:cite:`cls_2016` provides estimates of ENL for different Sentinel-1 products (average over all swaths), e.g. ENL of 4.4 for GRDH in
IW mode, and a description of the estimation process in section D1. The following shows a simple comparison between the
GRDH product:
@@ -68,7 +68,3 @@ consistently higher ENL values for the GRDH product (Avg. ENL: 4.81) in comparis
:alt: Figure 3: Scatter plot comparing ENL values between GRDH and NRB, calculated for selected areas (see Fig. 2).
Figure 3: Scatter plot comparing ENL values between GRDH and NRB, calculated for selected areas (see Fig. 2).
-
-References
-----------
-.. footbibliography::
\ No newline at end of file
diff --git a/docs/general/geoaccuracy.rst b/docs/general/geoaccuracy.rst
index 7dbe15b..010bd18 100644
--- a/docs/general/geoaccuracy.rst
+++ b/docs/general/geoaccuracy.rst
@@ -2,7 +2,7 @@ Geolocation Accuracy
====================
Item 4.3 of the CARD4L NRB specification requires, as minimum, an estimate of the absolute location error (ALE) "as
-bias and standard deviation, provided in slant range/azimuth, or Northing/Easting" [1]. As desired target the accuracy
+bias and standard deviation, provided in slant range/azimuth, or Northing/Easting" :cite:`ceos_2021`. As desired target the accuracy
is less or equal 0.1 pixels radial root mean square error (rRMSE), which can be defined as:
.. math::
@@ -32,7 +32,7 @@ Currently, the following simplifications need to be considered for the calculati
of each S1-NRB product:
* Processing induced errors (:math:`RMSE_{proc}`) and the error term related to DEM interpolation are not further considered and assumed to be 0.
-* The DEM accuracy (:math:`\sigma_{DEM}`) is estimated on the global mean accuracy LE90 reported for the COP-DEM [2] under the assumption of gaussian distribution:
+* The DEM accuracy (:math:`\sigma_{DEM}`) is estimated on the global mean accuracy LE90 reported for the COP-DEM :cite:`airbus_2022` under the assumption of gaussian distribution:
* Global: LE90 = 2.57; LE68 :math:`\approx` 1.56
@@ -41,8 +41,3 @@ of each S1-NRB product:
Development Status
------------------
The development status is tracked and discussed in the following Github issue: https://github.com/SAR-ARD/S1_NRB/issues/33
-
-References
------------
-* [1] https://ceos.org/ard/files/PFS/NRB/v5.5/CARD4L-PFS_NRB_v5.5.pdf
-* [2] https://spacedata.copernicus.eu/documents/20126/0/GEO1988-CopernicusDEM-SPE-002_ProductHandbook_I1.00.pdf/082dd479-f908-bf42-51bf-4c0053129f7c?t=1586526993604
diff --git a/docs/index.rst b/docs/index.rst
index 1907aa6..58ad608 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -7,10 +7,10 @@ Further information about these products can be found
`here `_.
.. toctree::
- :maxdepth: 2
+ :maxdepth: 2
- general/index.rst
- api.rst
- examples/index.rst
- about/index.rst
- tocs.rst
+ general/index.rst
+ api.rst
+ examples/index.rst
+ about/index.rst
+ tocs.rst
diff --git a/docs/references.bib b/docs/references.bib
index 58d1363..9ba60ce 100644
--- a/docs/references.bib
+++ b/docs/references.bib
@@ -31,4 +31,22 @@ @article{anfinsen.etal_2009
pages = {3795--3809},
issn = {0196-2892, 1558-0644},
doi = {10.1109/TGRS.2009.2019269}
+}
+@techreport{ceos_2021,
+ title = {Analysis {{Ready Data}} for {{Land}}: {{Normalized Radar Backscatter}}},
+ author = {CEOS},
+ institution = {CEOS},
+ date = {2021-10-14},
+ year = {2021},
+ number = {5.5},
+ url = {https://ceos.org/ard/files/PFS/NRB/v5.5/CARD4L-PFS_NRB_v5.5.pdf}
+}
+@techreport{airbus_2022,
+ title = {Copernicus {{DEM Product Handbook}}},
+ author = {Airbus},
+ institution = {Airbus},
+ date = {2022-11-29},
+ year = {2022},
+ number = {5.0},
+ url = {https://spacedata.copernicus.eu/documents/20123/122407/GEO1988-CopernicusDEM-SPE-002_ProductHandbook_I5.0+%281%29.pdf/706ee17d-2cce-f1fa-a73e-1686d28f09dd?t=1679657087883}
}
\ No newline at end of file
diff --git a/docs/tocs.rst b/docs/tocs.rst
index d46b839..1a44c8c 100644
--- a/docs/tocs.rst
+++ b/docs/tocs.rst
@@ -1,6 +1,8 @@
-Indices and tables
-==================
+.. only:: html
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`
+ Indices and tables
+ ==================
+
+ * :ref:`genindex`
+ * :ref:`modindex`
+ * :ref:`search`