In [1]:
Copied!
from cwl_loader import dump_cwl
from eoap_cwlwrap import wrap_locations
base_url = 'https://raw.githubusercontent.com/eoap/application-package-patterns/refs/heads/main'
workflow_id = 'pattern-1'
stage_in_cwl = f"{base_url}/templates/stage-in.cwl"
workflows_cwl = f"{base_url}/cwl-workflow/{workflow_id}.cwl"
stage_out_cwl = f"{base_url}/templates/stage-out.cwl"
main_workflow = wrap_locations(
directory_stage_in=stage_in_cwl,
workflows=workflows_cwl,
workflow_id=workflow_id,
stage_out=stage_out_cwl
)
from cwl_loader import dump_cwl
from eoap_cwlwrap import wrap_locations
base_url = 'https://raw.githubusercontent.com/eoap/application-package-patterns/refs/heads/main'
workflow_id = 'pattern-1'
stage_in_cwl = f"{base_url}/templates/stage-in.cwl"
workflows_cwl = f"{base_url}/cwl-workflow/{workflow_id}.cwl"
stage_out_cwl = f"{base_url}/templates/stage-out.cwl"
main_workflow = wrap_locations(
directory_stage_in=stage_in_cwl,
workflows=workflows_cwl,
workflow_id=workflow_id,
stage_out=stage_out_cwl
)
2025-09-20 17:22:43.749 | DEBUG | cwl_loader:load_cwl_from_location:213 - Loading CWL document from https://raw.githubusercontent.com/eoap/application-package-patterns/refs/heads/main/cwl-workflow/pattern-1.cwl...
2025-09-20 17:22:43.806 | DEBUG | cwl_loader:_load_cwl_from_stream:216 - Reading stream from https://raw.githubusercontent.com/eoap/application-package-patterns/refs/heads/main/cwl-workflow/pattern-1.cwl...
2025-09-20 17:22:43.827 | DEBUG | cwl_loader:load_cwl_from_stream:190 - CWL data of type <class 'ruamel.yaml.comments.CommentedMap'> successfully loaded from stream
2025-09-20 17:22:43.827 | DEBUG | cwl_loader:load_cwl_from_yaml:122 - Updating the model from version 'v1.0' to version 'v1.2'...
2025-09-20 17:22:43.828 | DEBUG | cwl_loader:load_cwl_from_yaml:133 - Raw CWL document successfully updated to v1.2!
2025-09-20 17:22:43.828 | DEBUG | cwl_loader:load_cwl_from_yaml:137 - Parsing the raw CWL document to the CWL Utils DOM...
2025-09-20 17:22:44.001 | DEBUG | cwl_loader:load_cwl_from_yaml:150 - Raw CWL document successfully parsed to the CWL Utils DOM!
2025-09-20 17:22:44.002 | DEBUG | cwl_loader:load_cwl_from_yaml:152 - Dereferencing the steps[].run...
2025-09-20 17:22:44.003 | DEBUG | cwl_loader:_on_process:70 - Checking if https://raw.githubusercontent.com/eoap/application-package-patterns/refs/heads/main/cwl-workflow/pattern-1.cwl#clt must be externally imported...
2025-09-20 17:22:44.003 | DEBUG | cwl_loader:_on_process:74 - run_url: https://raw.githubusercontent.com/eoap/application-package-patterns/refs/heads/main/cwl-workflow/pattern-1.cwl - uri: https://raw.githubusercontent.com/eoap/application-package-patterns/refs/heads/main/cwl-workflow/pattern-1.cwl
2025-09-20 17:22:44.004 | DEBUG | cwl_loader:load_cwl_from_yaml:159 - steps[].run successfully dereferenced! Now dereferencing the FQNs...
2025-09-20 17:22:44.004 | DEBUG | cwl_loader:load_cwl_from_yaml:163 - CWL document successfully dereferenced!
2025-09-20 17:22:44.005 | DEBUG | cwl_loader:load_cwl_from_yaml:166 - Sorting Process instances by dependencies....
2025-09-20 17:22:44.005 | DEBUG | cwl_loader:load_cwl_from_yaml:168 - Sorting process is over.
2025-09-20 17:22:44.006 | DEBUG | cwl_loader:_load_cwl_from_stream:224 - Stream from https://raw.githubusercontent.com/eoap/application-package-patterns/refs/heads/main/cwl-workflow/pattern-1.cwl successfully load!
2025-09-20 17:22:44.007 | DEBUG | cwl_loader:load_cwl_from_location:213 - Loading CWL document from https://raw.githubusercontent.com/eoap/application-package-patterns/refs/heads/main/templates/stage-out.cwl...
2025-09-20 17:22:44.058 | DEBUG | cwl_loader:_load_cwl_from_stream:216 - Reading stream from https://raw.githubusercontent.com/eoap/application-package-patterns/refs/heads/main/templates/stage-out.cwl...
2025-09-20 17:22:44.068 | DEBUG | cwl_loader:load_cwl_from_stream:190 - CWL data of type <class 'ruamel.yaml.comments.CommentedMap'> successfully loaded from stream
2025-09-20 17:22:44.069 | DEBUG | cwl_loader:load_cwl_from_yaml:122 - Updating the model from version 'v1.0' to version 'v1.2'...
2025-09-20 17:22:44.069 | DEBUG | cwl_loader:load_cwl_from_yaml:133 - Raw CWL document successfully updated to v1.2!
2025-09-20 17:22:44.070 | DEBUG | cwl_loader:load_cwl_from_yaml:137 - Parsing the raw CWL document to the CWL Utils DOM...
2025-09-20 17:22:44.476 | DEBUG | cwl_loader:load_cwl_from_yaml:150 - Raw CWL document successfully parsed to the CWL Utils DOM!
2025-09-20 17:22:44.477 | DEBUG | cwl_loader:load_cwl_from_yaml:152 - Dereferencing the steps[].run...
2025-09-20 17:22:44.477 | DEBUG | cwl_loader:load_cwl_from_yaml:159 - steps[].run successfully dereferenced! Now dereferencing the FQNs...
2025-09-20 17:22:44.478 | DEBUG | cwl_loader:load_cwl_from_yaml:163 - CWL document successfully dereferenced!
2025-09-20 17:22:44.478 | DEBUG | cwl_loader:load_cwl_from_yaml:166 - Sorting Process instances by dependencies....
2025-09-20 17:22:44.479 | DEBUG | cwl_loader:load_cwl_from_yaml:168 - Sorting process is over.
2025-09-20 17:22:44.479 | DEBUG | cwl_loader:_load_cwl_from_stream:224 - Stream from https://raw.githubusercontent.com/eoap/application-package-patterns/refs/heads/main/templates/stage-out.cwl successfully load!
2025-09-20 17:22:44.479 | DEBUG | eoap_cwlwrap:_load_process_from_location:499 - 'stage-out' from https://raw.githubusercontent.com/eoap/application-package-patterns/refs/heads/main/templates/stage-out.cwl is a valid single 'Process'
2025-09-20 17:22:44.480 | DEBUG | cwl_loader:load_cwl_from_location:213 - Loading CWL document from https://raw.githubusercontent.com/eoap/application-package-patterns/refs/heads/main/templates/stage-in.cwl...
2025-09-20 17:22:44.527 | DEBUG | cwl_loader:_load_cwl_from_stream:216 - Reading stream from https://raw.githubusercontent.com/eoap/application-package-patterns/refs/heads/main/templates/stage-in.cwl...
2025-09-20 17:22:44.534 | DEBUG | cwl_loader:load_cwl_from_stream:190 - CWL data of type <class 'ruamel.yaml.comments.CommentedMap'> successfully loaded from stream
2025-09-20 17:22:44.535 | DEBUG | cwl_loader:load_cwl_from_yaml:122 - Updating the model from version 'v1.0' to version 'v1.2'...
2025-09-20 17:22:44.535 | DEBUG | cwl_loader:load_cwl_from_yaml:133 - Raw CWL document successfully updated to v1.2!
2025-09-20 17:22:44.536 | DEBUG | cwl_loader:load_cwl_from_yaml:137 - Parsing the raw CWL document to the CWL Utils DOM...
2025-09-20 17:22:44.908 | DEBUG | cwl_loader:load_cwl_from_yaml:150 - Raw CWL document successfully parsed to the CWL Utils DOM!
2025-09-20 17:22:44.908 | DEBUG | cwl_loader:load_cwl_from_yaml:152 - Dereferencing the steps[].run...
2025-09-20 17:22:44.909 | DEBUG | cwl_loader:load_cwl_from_yaml:159 - steps[].run successfully dereferenced! Now dereferencing the FQNs...
2025-09-20 17:22:44.909 | DEBUG | cwl_loader:load_cwl_from_yaml:163 - CWL document successfully dereferenced!
2025-09-20 17:22:44.909 | DEBUG | cwl_loader:load_cwl_from_yaml:166 - Sorting Process instances by dependencies....
2025-09-20 17:22:44.910 | DEBUG | cwl_loader:load_cwl_from_yaml:168 - Sorting process is over.
2025-09-20 17:22:44.910 | DEBUG | cwl_loader:_load_cwl_from_stream:224 - Stream from https://raw.githubusercontent.com/eoap/application-package-patterns/refs/heads/main/templates/stage-in.cwl successfully load!
2025-09-20 17:22:44.911 | DEBUG | eoap_cwlwrap:_load_process_from_location:499 - 'directory-stage-in' from https://raw.githubusercontent.com/eoap/application-package-patterns/refs/heads/main/templates/stage-in.cwl is a valid single 'Process'
2025-09-20 17:22:44.911 | INFO | eoap_cwlwrap.types:_validate_stage_in:270 - Validating stage-in 'my-asthonishing-stage-in-directory'...
2025-09-20 17:22:44.912 | INFO | eoap_cwlwrap.types:_validate_stage_in:292 - stage-in 'my-asthonishing-stage-in-directory' is valid
2025-09-20 17:22:44.912 | INFO | eoap_cwlwrap.types:validate_stage_out:328 - Validating stage-out 'my-super-stage-out'...
2025-09-20 17:22:44.913 | INFO | eoap_cwlwrap.types:validate_stage_out:350 - stage-out 'my-super-stage-out' is valid
2025-09-20 17:22:44.913 | INFO | eoap_cwlwrap:_build_orchestrator_workflow:98 - Building the CWL Orchestrator Workflow...
2025-09-20 17:22:44.914 | INFO | eoap_cwlwrap:_build_orchestrator_workflow:127 - Analyzing pattern-1 inputs...
2025-09-20 17:22:44.914 | INFO | eoap_cwlwrap:_build_orchestrator_workflow:143 - * pattern-1/aoi: string
2025-09-20 17:22:44.915 | INFO | eoap_cwlwrap:_build_orchestrator_workflow:143 - * pattern-1/epsg: string
2025-09-20 17:22:44.915 | INFO | eoap_cwlwrap:_build_orchestrator_workflow:143 - * pattern-1/bands: string[]
2025-09-20 17:22:44.916 | INFO | eoap_cwlwrap:_build_orchestrator_workflow:143 - * pattern-1/item: Directory
2025-09-20 17:22:44.916 | INFO | eoap_cwlwrap:_build_orchestrator_workflow:156 - Directory type detected, creating a related 'directory_stage_in_0'...
2025-09-20 17:22:44.917 | INFO | eoap_cwlwrap:_build_orchestrator_workflow:158 - Converting Directory to URL-compatible type...
2025-09-20 17:22:44.917 | INFO | eoap_cwlwrap:_build_orchestrator_workflow:162 - Directory converted to Directory
2025-09-20 17:22:44.918 | INFO | eoap_cwlwrap:_build_orchestrator_workflow:203 - Connecting 'app/item' to 'directory_stage_in_0' output...
2025-09-20 17:22:44.918 | INFO | eoap_cwlwrap:_build_orchestrator_workflow:254 - Analyzing pattern-1 outputs...
2025-09-20 17:22:44.919 | INFO | eoap_cwlwrap:_build_orchestrator_workflow:260 - * pattern-1/water_bodies: Directory
2025-09-20 17:22:44.919 | INFO | eoap_cwlwrap:_build_orchestrator_workflow:265 - Directory type detected, creating a related 'stage_out_0'...
2025-09-20 17:22:44.920 | INFO | eoap_cwlwrap:_build_orchestrator_workflow:267 - Converting Directory to URL-compatible type...
2025-09-20 17:22:44.920 | INFO | eoap_cwlwrap:_build_orchestrator_workflow:271 - Directory converted to Directory
2025-09-20 17:22:44.922 | INFO | eoap_cwlwrap:_build_orchestrator_workflow:312 - Connecting 'app/water_bodies' to 'stage_out_0' output...
2025-09-20 17:22:44.923 | INFO | eoap_cwlwrap:_build_orchestrator_workflow:387 - Orchestrator Workflow built in 0.0098 seconds
Check the generated CWL Document¶
In [2]:
Copied!
import sys
dump_cwl(main_workflow, sys.stderr)
import sys
dump_cwl(main_workflow, sys.stderr)
cwlVersion: v1.2
$graph:
- id: main
class: Workflow
label: Workflow pattern-1 orchestrator
doc: This Workflow is used to orchestrate the Workflow pattern-1
inputs:
- id: aoi
label: area of interest - pattern-1/aoi
doc: area of interest as a bounding box - This parameter is derived from
pattern-1/aoi
default: -118.985,38.432,-118.183,38.938
type: string
- id: epsg
label: EPSG code - pattern-1/epsg
doc: EPSG code - This parameter is derived from pattern-1/epsg
default: EPSG:4326
type: string
- id: bands
label: bands used for the NDWI - pattern-1/bands
doc: bands used for the NDWI - This parameter is derived from
pattern-1/bands
default:
- green
- nir08
type:
name: _:50e2e80d-ccf0-4517-b584-ae604e0ef4b4
items: string
type: array
- id: another_input
label: Another Input - my-asthonishing-stage-in-directory/another_input
doc: An additional input for demonstration purposes - This parameter is
derived from my-asthonishing-stage-in-directory/another_input
type: string
- id: item
label: Landsat-8/9 acquisition reference - pattern-1/item
doc: Landsat-8/9 acquisition reference - This parameter is derived from
pattern-1/item
type: Directory
- id: s3_bucket
label: my-super-stage-out/s3_bucket
doc: 'This parameter is derived from: my-super-stage-out/s3_bucket'
type: string
- id: sub_path
label: my-super-stage-out/sub_path
doc: 'This parameter is derived from: my-super-stage-out/sub_path'
type: string
- id: aws_access_key_id
label: my-super-stage-out/aws_access_key_id
doc: 'This parameter is derived from: my-super-stage-out/aws_access_key_id'
type: string
- id: aws_secret_access_key
label: my-super-stage-out/aws_secret_access_key
doc: 'This parameter is derived from: my-super-stage-out/aws_secret_access_key'
type: string
- id: region_name
label: my-super-stage-out/region_name
doc: 'This parameter is derived from: my-super-stage-out/region_name'
type: string
- id: endpoint_url
label: my-super-stage-out/endpoint_url
doc: 'This parameter is derived from: my-super-stage-out/endpoint_url'
type: string
outputs:
- id: water_bodies
label: Water bodies detected
doc: Water bodies detected based on the NDWI and otsu threshold
outputSource:
- stage_out_0/s3_catalog_output
type: Directory
requirements:
- class: SubworkflowFeatureRequirement
- class: SchemaDefRequirement
types:
- $import:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml
steps:
- id: directory_stage_in_0
in:
- id: reference
source: item
- id: another_input
source: another_input
out:
- staged
run: '#my-asthonishing-stage-in-directory'
- id: app
in:
- id: aoi
source: aoi
- id: epsg
source: epsg
- id: bands
source: bands
- id: item
source: directory_stage_in_0/staged
out:
- water_bodies
run: '#pattern-1'
- id: stage_out_0
in:
- id: s3_bucket
source: s3_bucket
- id: sub_path
source: sub_path
- id: aws_access_key_id
source: aws_access_key_id
- id: aws_secret_access_key
source: aws_secret_access_key
- id: region_name
source: region_name
- id: endpoint_url
source: endpoint_url
- id: stac_catalog
source: app/water_bodies
out:
- s3_catalog_output
run: '#my-super-stage-out'
- id: my-asthonishing-stage-in-directory
class: CommandLineTool
inputs:
- id: reference
label: STAC Item URL
doc: A STAC Item to stage
type:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI
- id: another_input
label: Another Input
doc: An additional input for demonstration purposes
type: string
outputs:
- id: staged
type: Directory
outputBinding:
glob: .
requirements:
- class: NetworkAccess
networkAccess: true
- class: SchemaDefRequirement
types:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#Date
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#Date/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#DateTime
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#DateTime/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#Duration
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#Duration/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#Email
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#Email/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#Hostname
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#Hostname/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#IDNEmail
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#IDNEmail/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#IDNHostname
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#IDNHostname/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#IPv4
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#IPv4/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#IPv6
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#IPv6/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#IRI
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#IRI/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#IRIReference
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#IRIReference/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#JsonPointer
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#JsonPointer/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#Password
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#Password/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#RelativeJsonPointer
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#RelativeJsonPointer/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#UUID
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#UUID/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URIReference
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URIReference/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URITemplate
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URITemplate/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#Time
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#Time/value
type: string
type: record
- class: DockerRequirement
dockerPull: ghcr.io/eoap/mastering-app-package/stage:1.0.0
- class: InlineJavascriptRequirement
- class: InitialWorkDirRequirement
listing:
- entryname: stage.py
entry: |-
import pystac
import stac_asset
import asyncio
import os
import sys
config = stac_asset.Config(warn=True)
async def main(href: str):
item = pystac.read_file(href)
os.makedirs(item.id, exist_ok=True)
cwd = os.getcwd()
os.chdir(item.id)
item = await stac_asset.download_item(item=item, directory=".", config=config)
os.chdir(cwd)
cat = pystac.Catalog(
id="catalog",
description=f"catalog with staged {item.id}",
title=f"catalog with staged {item.id}",
)
cat.add_item(item)
cat.normalize_hrefs("./")
cat.save(catalog_type=pystac.CatalogType.SELF_CONTAINED)
return cat
href = sys.argv[1]
empty_arg = sys.argv[2]
cat = asyncio.run(main(href))
cwlVersion: v1.2
baseCommand:
- python
- stage.py
arguments:
- $( inputs.reference.value )
- $( inputs.another_input )
- id: pattern-1
class: Workflow
label: Water bodies detection based on NDWI and the otsu threshold
doc: Water bodies detection based on NDWI and otsu threshold applied to a
single Landsat-8/9 acquisition
inputs:
- id: aoi
label: area of interest
doc: area of interest as a bounding box
default: -118.985,38.432,-118.183,38.938
type: string
- id: epsg
label: EPSG code
doc: EPSG code
default: EPSG:4326
type: string
- id: bands
label: bands used for the NDWI
doc: bands used for the NDWI
default:
- green
- nir08
type:
name: _:50e2e80d-ccf0-4517-b584-ae604e0ef4b4
items: string
type: array
- id: item
label: Landsat-8/9 acquisition reference
doc: Landsat-8/9 acquisition reference
type: Directory
outputs:
- id: water_bodies
label: Water bodies detected
doc: Water bodies detected based on the NDWI and otsu threshold
outputSource:
- step/stac-catalog
type: Directory
requirements: []
cwlVersion: v1.2
steps:
- id: step
label: Water bodies detection
doc: Water bodies detection based on NDWI and otsu threshold applied to a
single Landsat-8/9 acquisition
in:
- id: item
source: item
- id: aoi
source: aoi
- id: epsg
source: epsg
- id: band
source: bands
out:
- stac-catalog
run: '#clt'
$namespaces: &id001
s: https://schema.org/
- id: my-super-stage-out
class: CommandLineTool
doc: Stage-out the results to S3
inputs:
- id: s3_bucket
type: string
- id: sub_path
type: string
- id: aws_access_key_id
type: string
- id: aws_secret_access_key
type: string
- id: region_name
type: string
- id: endpoint_url
type: string
- id: stac_catalog
label: STAC Catalog folder
doc: The folder containing the STAC catalog to stage out
type: Directory
outputs:
- id: s3_catalog_output
type:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI
outputBinding:
loadContents: true
glob: catalog-uri.txt
outputEval: |
${
return { "value": self[0].contents, "type": "https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI" };
}
requirements:
- class: NetworkAccess
networkAccess: true
- class: SchemaDefRequirement
types:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#Date
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#Date/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#DateTime
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#DateTime/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#Duration
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#Duration/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#Email
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#Email/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#Hostname
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#Hostname/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#IDNEmail
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#IDNEmail/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#IDNHostname
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#IDNHostname/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#IPv4
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#IPv4/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#IPv6
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#IPv6/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#IRI
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#IRI/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#IRIReference
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#IRIReference/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#JsonPointer
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#JsonPointer/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#Password
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#Password/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#RelativeJsonPointer
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#RelativeJsonPointer/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#UUID
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#UUID/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URIReference
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URIReference/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URITemplate
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URITemplate/value
type: string
type: record
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#Time
fields:
- name:
https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#Time/value
type: string
type: record
- class: DockerRequirement
dockerPull: ghcr.io/eoap/mastering-app-package/stage:1.0.0
- class: InlineJavascriptRequirement
- class: EnvVarRequirement
envDef:
- envName: aws_access_key_id
envValue: $( inputs.aws_access_key_id )
- envName: aws_secret_access_key
envValue: $( inputs.aws_secret_access_key )
- envName: aws_region_name
envValue: $( inputs.region_name )
- envName: aws_endpoint_url
envValue: $( inputs.endpoint_url )
- class: ResourceRequirement
- class: InitialWorkDirRequirement
listing:
- entryname: stage.py
entry: |-
import os
import sys
import pystac
import botocore
import boto3
import shutil
from pystac.stac_io import DefaultStacIO, StacIO
from urllib.parse import urlparse
cat_url = sys.argv[1]
bucket = sys.argv[2]
subfolder = sys.argv[3]
aws_access_key_id = os.environ["aws_access_key_id"]
aws_secret_access_key = os.environ["aws_secret_access_key"]
region_name = os.environ["aws_region_name"]
endpoint_url = os.environ["aws_endpoint_url"]
shutil.copytree(cat_url, "/tmp/catalog")
cat = pystac.read_file(os.path.join("/tmp/catalog", "catalog.json"))
class CustomStacIO(DefaultStacIO):
"""Custom STAC IO class that uses boto3 to read from S3."""
def __init__(self):
self.session = botocore.session.Session()
self.s3_client = self.session.create_client(
service_name="s3",
use_ssl=True,
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
endpoint_url=endpoint_url,
region_name=region_name,
)
def write_text(self, dest, txt, *args, **kwargs):
parsed = urlparse(dest)
if parsed.scheme == "s3":
self.s3_client.put_object(
Body=txt.encode("UTF-8"),
Bucket=parsed.netloc,
Key=parsed.path[1:],
ContentType="application/geo+json",
)
else:
super().write_text(dest, txt, *args, **kwargs)
# client = boto3.client(
# "s3",
# aws_access_key_id=aws_access_key_id,
# aws_secret_access_key=aws_secret_access_key,
# endpoint_url=endpoint_url,
# region_name=region_name,
# )
# StacIO.set_default(CustomStacIO)
# for item in cat.get_items():
# for key, asset in item.get_assets().items():
# s3_path = os.path.normpath(
# os.path.join(os.path.join(subfolder, item.id, asset.href))
# )
# print(f"upload {asset.href} to s3://{bucket}/{s3_path}",file=sys.stderr)
# client.upload_file(
# asset.get_absolute_href(),
# bucket,
# s3_path,
# )
# asset.href = f"s3://{bucket}/{s3_path}"
# item.add_asset(key, asset)
# cat.normalize_hrefs(f"s3://{bucket}/{subfolder}")
# for item in cat.get_items():
# # upload item to S3
# print(f"upload {item.id} to s3://{bucket}/{subfolder}", file=sys.stderr)
# pystac.write_file(item, item.get_self_href())
# # upload catalog to S3
# print(f"upload catalog.json to s3://{bucket}/{subfolder}", file=sys.stderr)
# pystac.write_file(cat, cat.get_self_href())
print(f"s3://{bucket}/{subfolder}/catalog.json", end="", file=sys.stdout)
cwlVersion: v1.2
baseCommand:
- python
- stage.py
arguments:
- $( inputs.stac_catalog.path )
- $( inputs.s3_bucket )
- ${ var firstPart = (Math.random() * 46656) | 0; var secondPart =
(Math.random() * 46656) | 0; firstPart = ("000" +
firstPart.toString(36)).slice(-3); secondPart = ("000" +
secondPart.toString(36)).slice(-3); return inputs.sub_path + "-" + firstPart
+ secondPart; }
stdout: catalog-uri.txt
- id: clt
class: CommandLineTool
inputs:
- id: item
type: Directory
inputBinding:
prefix: --input-item
- id: aoi
type: string
inputBinding:
prefix: --aoi
- id: epsg
type: string
inputBinding:
prefix: --epsg
- id: band
type:
- name: _:c0ae0700-7a38-45a5-a24c-4425f722f6a2
items: string
type: array
inputBinding:
prefix: --band
outputs:
- id: stac-catalog
type: Directory
outputBinding:
glob: .
requirements:
- class: InlineJavascriptRequirement
- class: EnvVarRequirement
envDef:
- envName: PATH
envValue: /app/envs/runner/bin
- class: ResourceRequirement
coresMax: 1
ramMax: 512
hints:
- class: DockerRequirement
dockerPull: ghcr.io/eoap/application-package-patterns/runner:0.2.0
cwlVersion: v1.2
baseCommand:
- runner
arguments:
- pattern-1
$namespaces: *id001