Skip to content

NEW: add upstream analysis outputs #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
gregcaporaso wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

NEW: add upstream analysis outputs #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 82 additions & 7 deletions q2_example_data/actions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@
SingleLanePerSamplePairedEndFastqDirFmt,
)
from q2_types.sample_data import SampleData
from q2_types.feature_data import (FeatureData, Sequence, Taxonomy,
TSVTaxonomyDirectoryFormat, DNASequencesDirectoryFormat)
from q2_types.feature_table import FeatureTable, Frequency, BIOMV210DirFmt
from qiime2 import Artifact, Metadata
from qiime2.metadata import MetadataFileError

Expand All @@ -29,6 +32,24 @@
GUT_TO_SOIL_DEMUX_SHA256 = (
"2e96e8a091e6b4ecf4635a10b18d3c4af2d2c4b98fd2772faeaf5bfe5f50b4a3"
)
GUT_TO_SOIL_FEATURE_TABLE_URL = (
"https://gut-to-soil-tutorial.readthedocs.io/en/2026.4/data/gut-to-soil/asv-table.qza"
)
GUT_TO_SOIL_FEATURE_TABLE_SHA256 = (
"98ebc5e3f3b784aa8283df68527aa132497fc4db6f5ae48fb4f89055ebbec63d"
)
GUT_TO_SOIL_FEATURE_DATA_SEQUENCE_URL = (
"https://gut-to-soil-tutorial.readthedocs.io/en/2026.4/data/gut-to-soil/asv-seqs.qza"
)
GUT_TO_SOIL_FEATURE_DATA_SEQUENCE_SHA256 = (
"bc7d1a9baf81eeea82444e7960d074e5ca08592efbf039e4dc0dbfa1119c10d7"
)
GUT_TO_SOIL_FEATURE_DATA_TAXONOMY_URL = (
"https://gut-to-soil-tutorial.readthedocs.io/en/2026.4/data/gut-to-soil/taxonomy.qza"
)
GUT_TO_SOIL_FEATURE_DATA_TAXONOMY_SHA256 = (
"6db1255c386c967878b67915c444593ce75ad61be6de6cc6ff7a080c1677eab9"
)

_DOWNLOAD_TIMEOUT_SECONDS = 60
_MAX_METADATA_DOWNLOAD_BYTES = 10 * 1024 * 1024
Expand Down Expand Up @@ -68,11 +89,11 @@ def _download_pinned_file(


def get_gut_to_soil_metadata() -> Metadata:
"""Download and validate the Gut-to-Soil tutorial sample metadata."""
"""Download and validate the gut-to-soil tutorial sample metadata."""
payload = _download_pinned_file(
url=GUT_TO_SOIL_METADATA_URL,
expected_sha256=GUT_TO_SOIL_METADATA_SHA256,
description="Gut-to-Soil sample metadata",
description="gut-to-soil sample metadata",
max_bytes=_MAX_METADATA_DOWNLOAD_BYTES,
)

Expand All @@ -83,16 +104,16 @@ def get_gut_to_soil_metadata() -> Metadata:
return Metadata.load(metadata_path)
except MetadataFileError as error:
raise RuntimeError(
"The downloaded Gut-to-Soil file is not valid QIIME 2 metadata."
"The downloaded gut-to-soil file is not valid QIIME 2 metadata."
) from error


def get_gut_to_soil_demux() -> SingleLanePerSamplePairedEndFastqDirFmt:
"""Download and validate the Gut-to-Soil demultiplexed sequences."""
"""Download and validate the gut-to-soil demultiplexed sequences."""
payload = _download_pinned_file(
url=GUT_TO_SOIL_DEMUX_URL,
expected_sha256=GUT_TO_SOIL_DEMUX_SHA256,
description="Gut-to-Soil demultiplexed sequences",
description="gut-to-soil demultiplexed sequences",
max_bytes=_MAX_DEMUX_DOWNLOAD_BYTES,
)

Expand All @@ -103,14 +124,68 @@ def get_gut_to_soil_demux() -> SingleLanePerSamplePairedEndFastqDirFmt:
artifact = Artifact.load(archive_path)
except (TypeError, ValueError, OSError) as error:
raise RuntimeError(
"The downloaded Gut-to-Soil demux file is not a valid QIIME 2 artifact."
"The downloaded gut-to-soil demux file is not a valid QIIME 2 artifact."
) from error

expected_type = SampleData[PairedEndSequencesWithQuality]
if artifact.type != expected_type:
raise RuntimeError(
"The downloaded Gut-to-Soil demux artifact has type "
"The downloaded gut-to-soil demux artifact has type "
f"{artifact.type}; expected {expected_type}."
)

return artifact.view(SingleLanePerSamplePairedEndFastqDirFmt)

def get_gut_to_soil_upstream_artifacts() -> (
BIOMV210DirFmt, DNASequencesDirectoryFormat, TSVTaxonomyDirectoryFormat):
payload_table = _download_pinned_file(
url=GUT_TO_SOIL_FEATURE_TABLE_URL,
expected_sha256=GUT_TO_SOIL_FEATURE_TABLE_SHA256,
description="gut-to-soil asv table",
max_bytes=_MAX_DEMUX_DOWNLOAD_BYTES,
)
payload_table_expected_type = FeatureTable[Frequency]
payload_table_target_dirfmt = BIOMV210DirFmt

payload_sequences = _download_pinned_file(
url=GUT_TO_SOIL_FEATURE_DATA_SEQUENCE_URL,
expected_sha256=GUT_TO_SOIL_FEATURE_DATA_SEQUENCE_SHA256,
description="gut-to-soil asv sequences",
max_bytes=_MAX_DEMUX_DOWNLOAD_BYTES,
)
payload_sequences_expected_type = FeatureData[Sequence]
payload_sequences_target_dirfmt = DNASequencesDirectoryFormat

payload_taxonomy = _download_pinned_file(
url=GUT_TO_SOIL_FEATURE_DATA_TAXONOMY_URL,
expected_sha256=GUT_TO_SOIL_FEATURE_DATA_TAXONOMY_SHA256,
description="gut-to-soil asv taxonomy",
max_bytes=_MAX_DEMUX_DOWNLOAD_BYTES,
)
payload_taxonomy_expected_type = FeatureData[Taxonomy]
payload_taxonomy_target_dirfmt = TSVTaxonomyDirectoryFormat

results = []
for payload, expected_type, target_dirfmt in (
[payload_table, payload_table_expected_type, payload_table_target_dirfmt],
[payload_sequences, payload_sequences_expected_type, payload_sequences_target_dirfmt],
[payload_taxonomy, payload_taxonomy_expected_type, payload_taxonomy_target_dirfmt]):
with TemporaryDirectory() as temp_dir:
archive_path = Path(temp_dir) / "artifact.qza"
archive_path.write_bytes(payload)
try:
artifact = Artifact.load(archive_path)
except (TypeError, ValueError, OSError) as error:
raise RuntimeError(
"The downloaded file is not a valid QIIME 2 artifact."
) from error

if artifact.type != expected_type:
raise RuntimeError(
"The downloaded artifact has type "
f"{artifact.type}; {expected_type} was expected."
)

results.append(artifact.view(target_dirfmt))

return tuple(results)
26 changes: 26 additions & 0 deletions q2_example_data/citations.bib
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1,27 @@
% Add BibTeX entries for software and papers cited by this plugin.
@ARTICLE{Meilander2025-pc,
title = "{Upcycling human excrement: the gut microbiome to soil microbiome
axis}",
author = "Meilander, Jeff and Herman, Chloe and Manley, Andrew and
Augustine, Georgia and Birdsell, Dawn and Bolyen, Evan and
Celona, Kimberly R and Coffey, Hayden and Cocking, Jill and
Donoghue, Teddy and Draves, Alexis and Erickson, Daryn and Foley,
Marissa and Gehret, Liz and Hagen, Johannah and Hepp, Crystal and
Ingram, Parker and John, David and Kadar, Katarina and Keim, Paul
and Lloyd, Victoria and Osterink, Christina and Monsaint-Queeney,
Victoria and Ramirez, Diego and Romero, Antonio and Ruby, Megan C
and Sahl, Jason W and Soloway, Sydni and Stone, Nathan E and
Trottier, Shannon and Van Orden, Kaleb and Painter, Alexis and
Wallace, Sam and Wilcox, Larissa and Wood, Colin V and Yancey,
Jaiden and Caporaso, J Gregory",
journal = "ISME Commun.",
publisher = "Oxford University Press (OUP)",
volume = 5,
number = 1,
pages = "ycaf089",
month = jan,
year = 2025,
keywords = "16S; amplicon; biosolids; compost; compost toilet; human
excrement; latrines; manure; microbiome; soil; sustainability",
language = "en"
}
37 changes: 31 additions & 6 deletions q2_example_data/plugin_setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,19 @@
from q2_types.metadata import ImmutableMetadata
from q2_types.per_sample_sequences import PairedEndSequencesWithQuality
from q2_types.sample_data import SampleData
from q2_types.feature_data import FeatureData, Sequence, Taxonomy
from q2_types.feature_table import FeatureTable, Frequency
from qiime2.plugin import Plugin

from q2_example_data import __version__
from q2_example_data.actions import (
get_gut_to_soil_demux,
get_gut_to_soil_metadata,
get_gut_to_soil_upstream_artifacts
)

_G2S_TUTORIAL_URL = "https://amplicon-docs.qiime2.org/en/stable/tutorials/gut-to-soil/"

plugin = Plugin(
name="example-data",
version=__version__,
Expand All @@ -28,12 +33,12 @@
("metadata", ImmutableMetadata),
],
output_descriptions={
"metadata": "Sample metadata for the Gut-to-Soil tutorial dataset.",
"metadata": "The sample metadata used in the gut-to-soil tutorial",
},
input_descriptions={},
parameter_descriptions={},
name="Get Gut-to-Soil sample metadata",
description="Download the Gut-to-Soil tutorial sample metadata, verify the pinned content, and return it as an immutable metadata artifact.",
name="Get gut-to-soil sample metadata",
description=f"Download the gut-to-soil tutorial sample metadata, verify the pinned content, and return it as an immutable metadata artifact. For usage examples, see {_G2S_TUTORIAL_URL}.",
)

plugin.methods.register_function(
Expand All @@ -44,10 +49,30 @@
("demux", SampleData[PairedEndSequencesWithQuality]),
],
output_descriptions={
"demux": "Demultiplexed paired-end sequences for the Gut-to-Soil tutorial dataset.",
"demux": "The demultiplexed paired-end sequences used in the gut-to-soil tutorial.",
},
input_descriptions={},
parameter_descriptions={},
name="Get gut-to-soil demultiplexed sequences",
description=f"Download the gut-to-soil tutorial demultiplexed sequences, verify the pinned archive, and return the corresponding artifact. For usage examples, see {_G2S_TUTORIAL_URL}.",
)

plugin.methods.register_function(
function=get_gut_to_soil_upstream_artifacts,
inputs={},
parameters={},
outputs=[
("asv_table", FeatureTable[Frequency]),
("asv_sequences", FeatureData[Sequence]),
("asv_taxonomy", FeatureData[Taxonomy]),
],
output_descriptions={
"asv_table": "The ASV table generated in the gut-to-soil tutorial, as output from DADA2.",
"asv_sequences": "The ASV sequences generated in the gut-to-soil tutorial, as output from DADA2.",
"asv_taxonomy": "The ASV taxonomy generated in the gut-to-soil tutorial.",
},
input_descriptions={},
parameter_descriptions={},
name="Get Gut-to-Soil demultiplexed sequences",
description="Download the Gut-to-Soil tutorial demultiplexed sequences, verify the pinned QIIME 2 archive, and return paired-end sequences with quality scores.",
name="Get gut-to-soil upstream artifacts.",
description=f"Download the three primary outputs of the upstream analysis steps of the gut-to-soil tutorial and verify their checksums. For usage examples, see {_G2S_TUTORIAL_URL}.",
)
Loading