{ "cells": [ { "cell_type": "markdown", "id": "c0aa34e4-0f2e-42d0-8ee1-f48c1d87360c", "metadata": { "pycharm": { "name": "#%% md\n" } }, "source": [ "## Nextflow with SLURM Tutorial\n", "\n", "Let's run a Nextflow pipeline." ] }, { "cell_type": "code", "execution_count": 1, "id": "c07b4551-4802-4338-b66f-9cc041e62796", "metadata": { "pycharm": { "name": "#%%\n" } }, "outputs": [], "source": [ "module load pcluster-helpers" ] }, { "cell_type": "code", "execution_count": 2, "id": "f79d1d9e-f0a0-4e4f-90ca-33f5c356b646", "metadata": { "pycharm": { "name": "#%%\n" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\u001b[1m \u001b[0m\n", "\u001b[1m \u001b[0m\u001b[1;33mUsage: \u001b[0m\u001b[1mpcluster-helper [OPTIONS] COMMAND [ARGS]...\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m\n", "\u001b[1m \u001b[0m\n", "\u001b[2m╭─\u001b[0m\u001b[2m Options \u001b[0m\u001b[2m───────────────────────────────────────────────────────────────────\u001b[0m\u001b[2m─╮\u001b[0m\n", "\u001b[2m│\u001b[0m \u001b[1;36m-\u001b[0m\u001b[1;36m-install\u001b[0m\u001b[1;36m-completion\u001b[0m \u001b[1;2;33m[\u001b[0m\u001b[1;33mbash\u001b[0m\u001b[1;2;33m|\u001b[0m\u001b[1;33mzsh\u001b[0m\u001b[1;2;33m|\u001b[0m\u001b[1;33mfish\u001b[0m\u001b[1;2;33m|\u001b[0m\u001b[1;33mpowershe\u001b[0m Install completion for \u001b[2m│\u001b[0m\n", "\u001b[2m│\u001b[0m \u001b[1;33mll\u001b[0m\u001b[1;2;33m|\u001b[0m\u001b[1;33mpwsh\u001b[0m\u001b[1;2;33m]\u001b[0m\u001b[1;33m \u001b[0m the specified shell. \u001b[2m│\u001b[0m\n", "\u001b[2m│\u001b[0m \u001b[2m[default: None] \u001b[0m \u001b[2m│\u001b[0m\n", "\u001b[2m│\u001b[0m \u001b[1;36m-\u001b[0m\u001b[1;36m-show\u001b[0m\u001b[1;36m-completion\u001b[0m \u001b[1;2;33m[\u001b[0m\u001b[1;33mbash\u001b[0m\u001b[1;2;33m|\u001b[0m\u001b[1;33mzsh\u001b[0m\u001b[1;2;33m|\u001b[0m\u001b[1;33mfish\u001b[0m\u001b[1;2;33m|\u001b[0m\u001b[1;33mpowershe\u001b[0m Show completion for the \u001b[2m│\u001b[0m\n", "\u001b[2m│\u001b[0m \u001b[1;33mll\u001b[0m\u001b[1;2;33m|\u001b[0m\u001b[1;33mpwsh\u001b[0m\u001b[1;2;33m]\u001b[0m\u001b[1;33m \u001b[0m specified shell, to \u001b[2m│\u001b[0m\n", "\u001b[2m│\u001b[0m copy it or customize \u001b[2m│\u001b[0m\n", "\u001b[2m│\u001b[0m the installation. \u001b[2m│\u001b[0m\n", "\u001b[2m│\u001b[0m \u001b[2m[default: None] \u001b[0m \u001b[2m│\u001b[0m\n", "\u001b[2m│\u001b[0m \u001b[1;36m-\u001b[0m\u001b[1;36m-help\u001b[0m \u001b[1;33m \u001b[0m Show this message and \u001b[2m│\u001b[0m\n", "\u001b[2m│\u001b[0m exit. \u001b[2m│\u001b[0m\n", "\u001b[2m╰──────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", "\u001b[2m╭─\u001b[0m\u001b[2m Commands \u001b[0m\u001b[2m──────────────────────────────────────────────────────────────────\u001b[0m\u001b[2m─╮\u001b[0m\n", "\u001b[2m│\u001b[0m \u001b[1;36mgen-nxf-slurm-config\u001b[0m\u001b[1;36m \u001b[0m Generate a slurm.config for nextflow that is \u001b[2m│\u001b[0m\n", "\u001b[2m│\u001b[0m \u001b[1;36m \u001b[0m compatible with your cluster. \u001b[2m│\u001b[0m\n", "\u001b[2m│\u001b[0m \u001b[1;36msinfo \u001b[0m\u001b[1;36m \u001b[0m A more helpful sinfo \u001b[2m│\u001b[0m\n", "\u001b[2m╰──────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", "\n" ] } ], "source": [ "pcluster-helper --help" ] }, { "cell_type": "markdown", "id": "059c46d9-1533-4b40-938b-8698e07f6fd1", "metadata": { "pycharm": { "name": "#%% md\n" } }, "source": [ "### Generate a Nextflow slurm.config\n", "\n", "We'll use the `pcluster-helper gen-nxf-slurm-config` in order to generate a default slurm configuration file." ] }, { "cell_type": "code", "execution_count": 3, "id": "0ef18353-c9a4-4c56-ac06-e30b2b0b1520", "metadata": { "pycharm": { "name": "#%%\n" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\u001b[1m \u001b[0m\n", "\u001b[1m \u001b[0m\u001b[1;33mUsage: \u001b[0m\u001b[1mpcluster-helper gen-nxf-slurm-config [OPTIONS]\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m\n", "\u001b[1m \u001b[0m\n", " Generate a slurm.config for nextflow that is compatible with your cluster. \n", " \u001b[2mYou will see a process label for each partition/node type.\u001b[0m \n", " \u001b[2mUse the configuration in your process by setting the label to match the label \u001b[0m \n", " \u001b[2min the config.\u001b[0m \n", " \n", "\u001b[2m╭─\u001b[0m\u001b[2m Options \u001b[0m\u001b[2m───────────────────────────────────────────────────────────────────\u001b[0m\u001b[2m─╮\u001b[0m\n", "\u001b[2m│\u001b[0m \u001b[1;36m-\u001b[0m\u001b[1;36m-include\u001b[0m\u001b[1;36m-memory\u001b[0m \u001b[1;35m-\u001b[0m\u001b[1;35m-no\u001b[0m\u001b[1;35m-include-memory\u001b[0m \u001b[1;33m \u001b[0m Include scheduleable \u001b[2m│\u001b[0m\n", "\u001b[2m│\u001b[0m memory \u001b[2m│\u001b[0m\n", "\u001b[2m│\u001b[0m \u001b[2m[default: \u001b[0m \u001b[2m│\u001b[0m\n", "\u001b[2m│\u001b[0m \u001b[2mno-include-memory] \u001b[0m \u001b[2m│\u001b[0m\n", "\u001b[2m│\u001b[0m \u001b[1;36m-\u001b[0m\u001b[1;36m-scheduleable\u001b[0m\u001b[1;36m-memory\u001b[0m \u001b[1;33mFLOAT\u001b[0m Schedulable amount of \u001b[2m│\u001b[0m\n", "\u001b[2m│\u001b[0m memory. Default is \u001b[2m│\u001b[0m\n", "\u001b[2m│\u001b[0m 95% \u001b[2m│\u001b[0m\n", "\u001b[2m│\u001b[0m \u001b[2m[default: 0.95] \u001b[0m \u001b[2m│\u001b[0m\n", "\u001b[2m│\u001b[0m \u001b[1;36m-\u001b[0m\u001b[1;36m-output\u001b[0m \u001b[1;33mTEXT \u001b[0m Path to nextflow \u001b[2m│\u001b[0m\n", "\u001b[2m│\u001b[0m slurm config file. \u001b[2m│\u001b[0m\n", "\u001b[2m│\u001b[0m \u001b[2m[default: None] \u001b[0m \u001b[2m│\u001b[0m\n", "\u001b[2m│\u001b[0m \u001b[1;36m-\u001b[0m\u001b[1;36m-overwrite\u001b[0m \u001b[1;35m-\u001b[0m\u001b[1;35m-no\u001b[0m\u001b[1;35m-overwrite\u001b[0m \u001b[1;33m \u001b[0m Overwrite existing \u001b[2m│\u001b[0m\n", "\u001b[2m│\u001b[0m file. \u001b[2m│\u001b[0m\n", "\u001b[2m│\u001b[0m \u001b[2m[default: \u001b[0m \u001b[2m│\u001b[0m\n", "\u001b[2m│\u001b[0m \u001b[2mno-overwrite] \u001b[0m \u001b[2m│\u001b[0m\n", "\u001b[2m│\u001b[0m \u001b[1;36m-\u001b[0m\u001b[1;36m-stdout\u001b[0m \u001b[1;35m-\u001b[0m\u001b[1;35m-no\u001b[0m\u001b[1;35m-stdout\u001b[0m \u001b[1;33m \u001b[0m Write slurm config to \u001b[2m│\u001b[0m\n", "\u001b[2m│\u001b[0m stdout \u001b[2m│\u001b[0m\n", "\u001b[2m│\u001b[0m \u001b[2m[default: no-stdout] \u001b[0m \u001b[2m│\u001b[0m\n", "\u001b[2m│\u001b[0m \u001b[1;36m-\u001b[0m\u001b[1;36m-help\u001b[0m \u001b[1;33m \u001b[0m Show this message and \u001b[2m│\u001b[0m\n", "\u001b[2m│\u001b[0m exit. \u001b[2m│\u001b[0m\n", "\u001b[2m╰──────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", "\n" ] } ], "source": [ "pcluster-helper gen-nxf-slurm-config --help" ] }, { "cell_type": "markdown", "id": "460f0a58-c3bf-4ef1-973f-67fdd733c112", "metadata": {}, "source": [ "### One time Nextflow SLURM Setup\n", "\n", "\n", "If you want Nextflow to distribute your jobs using the SLURM cluster you'll need to generate a SLURM executor config that Nextflow understands.\n", "\n", "You can generate this once, and continue to use it for all Nextflow pipelines.\n", "\n", "```bash\n", "pcluster-helper \\\n", " gen-nxf-slurm-config \\\n", " --include-memory \\\n", " --output ~/slurm.config \\\n", " --overwrite\n", "```" ] }, { "cell_type": "markdown", "id": "6e95e37c-d4bb-41ff-b3d6-a31eac4ca990", "metadata": { "pycharm": { "name": "#%% md\n" } }, "source": [ "We'll also want to create a default configuration for jobs that don't have a process tag. I'll choose a small one for this demonstration, but you should choose which instance is best for your workflows." ] }, { "cell_type": "code", "execution_count": 4, "id": "8044ffda-06e3-4a09-b236-717d990b2774", "metadata": { "pycharm": { "name": "#%%\n" } }, "outputs": [], "source": [ "cat > ./slurm-default.config <<'EOF'\n", "process {\n", " executor='slurm'\n", " queue = 'dev'\n", " cpus = 8\n", "\n", " memory = '30GB'\n", " clusterOptions = '--exclusive --constraint m5a2xlarge'\n", "}\n", "EOF" ] }, { "cell_type": "code", "execution_count": 5, "id": "acf12e92-8ede-41af-8dff-8e61b265141a", "metadata": {}, "outputs": [], "source": [ "# cleanup\n", "rm -rf test.config\n", "rm -rf test.config.*\n", "rm -rf samplesheet_test.csv\n", "rm -rf samplesheet_test.csv.*\n", "#rm -rf .nextflow\n", "#sleep 1m\n", "#rm -rf .nextflow*\n", "#rm -rf work\n", "#rm -rf results" ] }, { "cell_type": "code", "execution_count": 6, "id": "ea2b3977-133d-4c84-8b22-4ced8eaebc9e", "metadata": { "pycharm": { "name": "#%%\n" } }, "outputs": [], "source": [ "wget --quiet https://raw.githubusercontent.com/nf-core/rnaseq/master/conf/test.config" ] }, { "cell_type": "code", "execution_count": 7, "id": "df145ad7-598d-4589-a91c-f78880357cff", "metadata": { "pycharm": { "name": "#%%\n" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2\n" ] } ], "source": [ "wget --quiet https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/samplesheet/v3.4/samplesheet_test.csv\n", "cat samplesheet_test.csv |wc -l" ] }, { "cell_type": "code", "execution_count": 8, "id": "74cdd94a-7c07-42a6-9b08-02292bc88926", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "sample,fastq_1,fastq_2,strandedness\n", "WT_REP1,s3://nf-core-awsmegatests/rnaseq/input_data/minimal/GSE110004/SRR6357070_1.fastq.gz,s3://nf-core-awsmegatests/rnaseq/input_data/minimal/GSE110004/SRR6357070_2.fastq.gz,reverse\n" ] } ], "source": [ "head -n 3 samplesheet_test.csv " ] }, { "cell_type": "markdown", "id": "b1d669fa-7b87-45ce-b2cd-4721b1783be7", "metadata": { "pycharm": { "name": "#%% md\n" } }, "source": [ "### Reduce the number of samples\n", "\n", "The samplesheet has 8 rows, and I don't want to actually run 8. I'll run the first sample." ] }, { "cell_type": "code", "execution_count": 9, "id": "c78d280b-cd96-4dfe-934c-5c6e8de18015", "metadata": { "pycharm": { "name": "#%%\n" } }, "outputs": [], "source": [ "cat samplesheet_test.csv |head -n 2 > samplesheet_test_t.csv ; mv samplesheet_test_t.csv samplesheet_test.csv" ] }, { "cell_type": "code", "execution_count": 10, "id": "26e7b44c-16c5-4932-bd00-9c009f94095e", "metadata": { "pycharm": { "name": "#%%\n" } }, "outputs": [], "source": [ "module load nextflow" ] }, { "cell_type": "code", "execution_count": 11, "id": "1b1b8f39-6f83-4250-9086-c6b1da5e47bc", "metadata": { "pycharm": { "name": "#%%\n" } }, "outputs": [], "source": [ "#nextflow -h" ] }, { "cell_type": "code", "execution_count": 12, "id": "e5c6a1d0-efa9-45f2-b966-1b5553e86f08", "metadata": { "pycharm": { "name": "#%%\n" } }, "outputs": [], "source": [ "#nextflow run -h" ] }, { "cell_type": "code", "execution_count": 13, "id": "b3862296-866d-4467-90a8-da9493093dd8", "metadata": { "pycharm": { "name": "#%%\n" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "N E X T F L O W ~ version 22.10.1\n", "WARN: It appears you have never run this project before -- Option `-resume` is ignored\n", "Launching `https://github.com/nf-core/rnaseq` [jovial_bohr] DSL2 - revision: e049f51f02 [master]\n", "\n", "\n", "-\u001b[2m----------------------------------------------------\u001b[0m-\n", " \u001b[0;32m,--.\u001b[0;30m/\u001b[0;32m,-.\u001b[0m\n", "\u001b[0;34m ___ __ __ __ ___ \u001b[0;32m/,-._.--~'\u001b[0m\n", "\u001b[0;34m |\\ | |__ __ / ` / \\ |__) |__ \u001b[0;33m} {\u001b[0m\n", "\u001b[0;34m | \\| | \\__, \\__/ | \\ |___ \u001b[0;32m\\`-._,-`-,\u001b[0m\n", " \u001b[0;32m`._,._,'\u001b[0m\n", "\u001b[0;35m nf-core/rnaseq v3.9\u001b[0m\n", "-\u001b[2m----------------------------------------------------\u001b[0m-\n", "\u001b[1mCore Nextflow options\u001b[0m\n", " \u001b[0;34mrevision : \u001b[0;32mmaster\u001b[0m\n", " \u001b[0;34mrunName : \u001b[0;32mjovial_bohr\u001b[0m\n", " \u001b[0;34mcontainerEngine : \u001b[0;32msingularity\u001b[0m\n", " \u001b[0;34mlaunchDir : \u001b[0;32m/home/jillian/bioanalyze-hpc/docs/workflow-managers/nextflow\u001b[0m\n", " \u001b[0;34mworkDir : \u001b[0;32m/home/jillian/bioanalyze-hpc/docs/workflow-managers/nextflow/work\u001b[0m\n", " \u001b[0;34mprojectDir : \u001b[0;32m/home/jillian/.nextflow/assets/nf-core/rnaseq\u001b[0m\n", " \u001b[0;34muserName : \u001b[0;32mjillian\u001b[0m\n", " \u001b[0;34mprofile : \u001b[0;32mslurm\u001b[0m\n", " \u001b[0;34mconfigFiles : \u001b[0;32m/home/jillian/.nextflow/assets/nf-core/rnaseq/nextflow.config, /home/jillian/bioanalyze-hpc/docs/workflow-managers/nextflow/test.config, /home/jillian/bioanalyze-hpc/docs/workflow-managers/nextflow/slurm-default.config, /home/jillian/slurm.config\u001b[0m\n", "\n", "\u001b[1mInput/output options\u001b[0m\n", " \u001b[0;34minput : \u001b[0;32m./samplesheet_test.csv\u001b[0m\n", " \u001b[0;34moutdir : \u001b[0;32m./results\u001b[0m\n", "\n", "\u001b[1mUMI options\u001b[0m\n", " \u001b[0;34mumitools_bc_pattern : \u001b[0;32mNNNN\u001b[0m\n", " \u001b[0;34mumitools_bc_pattern2 : \u001b[0;32mnull\u001b[0m\n", " \u001b[0;34mumitools_umi_separator : \u001b[0;32mnull\u001b[0m\n", "\n", "\u001b[1mRead filtering options\u001b[0m\n", " \u001b[0;34mbbsplit_fasta_list : \u001b[0;32mhttps://github.com/nf-core/test-datasets/raw/rnaseq/reference/bbsplit_fasta_list.txt\u001b[0m\n", " \u001b[0;34mskip_bbsplit : \u001b[0;32mfalse\u001b[0m\n", "\n", "\u001b[1mReference genome options\u001b[0m\n", " \u001b[0;34mfasta : \u001b[0;32mhttps://github.com/nf-core/test-datasets/raw/rnaseq/reference/genome.fasta\u001b[0m\n", " \u001b[0;34mgtf : \u001b[0;32mhttps://github.com/nf-core/test-datasets/raw/rnaseq/reference/genes.gtf.gz\u001b[0m\n", " \u001b[0;34mgff : \u001b[0;32mhttps://github.com/nf-core/test-datasets/raw/rnaseq/reference/genes.gff.gz\u001b[0m\n", " \u001b[0;34mtranscript_fasta : \u001b[0;32mhttps://github.com/nf-core/test-datasets/raw/rnaseq/reference/transcriptome.fasta\u001b[0m\n", " \u001b[0;34madditional_fasta : \u001b[0;32mhttps://github.com/nf-core/test-datasets/raw/rnaseq/reference/gfp.fa.gz\u001b[0m\n", " \u001b[0;34mhisat2_index : \u001b[0;32mhttps://github.com/nf-core/test-datasets/raw/rnaseq/reference/hisat2.tar.gz\u001b[0m\n", " \u001b[0;34mrsem_index : \u001b[0;32mhttps://github.com/nf-core/test-datasets/raw/rnaseq/reference/rsem.tar.gz\u001b[0m\n", " \u001b[0;34msalmon_index : \u001b[0;32mhttps://github.com/nf-core/test-datasets/raw/rnaseq/reference/salmon.tar.gz\u001b[0m\n", "\n", "\u001b[1mAlignment options\u001b[0m\n", " \u001b[0;34mpseudo_aligner : \u001b[0;32msalmon\u001b[0m\n", "\n", "\u001b[1mInstitutional config options\u001b[0m\n", " \u001b[0;34mconfig_profile_name : \u001b[0;32mTest profile\u001b[0m\n", " \u001b[0;34mconfig_profile_description: \u001b[0;32mMinimal test dataset to check pipeline function\u001b[0m\n", "\n", "\u001b[1mMax job request options\u001b[0m\n", " \u001b[0;34mmax_cpus : \u001b[0;32m2\u001b[0m\n", " \u001b[0;34mmax_memory : \u001b[0;32m6.GB\u001b[0m\n", " \u001b[0;34mmax_time : \u001b[0;32m6.h\u001b[0m\n", "\n", "!! Only displaying parameters that differ from the pipeline defaults !!\n", "-\u001b[2m----------------------------------------------------\u001b[0m-\n", "If you use nf-core/rnaseq for your analysis please cite:\n", "\n", "* The pipeline\n", " https://doi.org/10.5281/zenodo.1400710\n", "\n", "* The nf-core framework\n", " https://doi.org/10.1038/s41587-020-0439-x\n", "\n", "* Software dependencies\n", " https://github.com/nf-core/rnaseq/blob/master/CITATIONS.md\n", "-\u001b[2m----------------------------------------------------\u001b[0m-\n", "WARN: =============================================================================\n", " Both '--gtf' and '--gff' parameters have been provided.\n", " Using GTF file as priority.\n", "===================================================================================\n", "WARN: =============================================================================\n", " '--transcript_fasta' parameter has been provided.\n", " Make sure transcript names in this file match those in the GFF/GTF file.\n", "\n", " Please see:\n", " https://github.com/nf-core/rnaseq/issues/753\n", "===================================================================================\n", "[1b/5d1920] Submitted process > NFCORE_RNASEQ:RNASEQ:INPUT_CHECK:SAMPLESHEET_CHECK (samplesheet_test.csv)\n", "[da/003916] Submitted process > NFCORE_RNASEQ:RNASEQ:PREPARE_GENOME:GUNZIP_GTF (genes.gtf.gz)\n", "[cc/344523] Submitted process > NFCORE_RNASEQ:RNASEQ:PREPARE_GENOME:GUNZIP_ADDITIONAL_FASTA (gfp.fa.gz)\n", "[b6/df43a5] Submitted process > NFCORE_RNASEQ:RNASEQ:PREPARE_GENOME:UNTAR_SALMON_INDEX (salmon.tar.gz)\n", "[6c/37bd75] Submitted process > NFCORE_RNASEQ:RNASEQ:PREPARE_GENOME:CAT_ADDITIONAL_FASTA (gfp.fa)\n", "[0e/d50a18] Submitted process > NFCORE_RNASEQ:RNASEQ:FASTQC_UMITOOLS_TRIMGALORE:FASTQC (WT_REP1)\n", "[55/8956ae] Submitted process > NFCORE_RNASEQ:RNASEQ:FASTQC_UMITOOLS_TRIMGALORE:TRIMGALORE (WT_REP1)\n", "[d1/dada77] Submitted process > NFCORE_RNASEQ:RNASEQ:PREPARE_GENOME:CUSTOM_GETCHROMSIZES (genome_gfp.fasta)\n", "[83/04dcf8] Submitted process > NFCORE_RNASEQ:RNASEQ:PREPARE_GENOME:GTF2BED (genome_gfp.gtf)\n", "[6a/8038cd] Submitted process > NFCORE_RNASEQ:RNASEQ:PREPARE_GENOME:STAR_GENOMEGENERATE (genome_gfp.fasta)\n", "[9b/ef0f74] Submitted process > NFCORE_RNASEQ:RNASEQ:PREPARE_GENOME:BBMAP_BBSPLIT\n", "[92/3a1c8d] Submitted process > NFCORE_RNASEQ:RNASEQ:BBMAP_BBSPLIT (1)\n", "[a1/735443] Submitted process > NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN (WT_REP1)\n", "[92/012f9f] Submitted process > NFCORE_RNASEQ:RNASEQ:QUANTIFY_SALMON:SALMON_QUANT (WT_REP1)\n", "[64/bb07e6] Submitted process > NFCORE_RNASEQ:RNASEQ:QUANTIFY_SALMON:SALMON_TX2GENE (genome_gfp.gtf)\n", "[a8/a7ca66] Submitted process > NFCORE_RNASEQ:RNASEQ:QUANTIFY_SALMON:SALMON_TXIMPORT\n", "[71/1db4af] Submitted process > NFCORE_RNASEQ:RNASEQ:QUANTIFY_SALMON:SALMON_SE_GENE (salmon_tx2gene.tsv)\n", "[f4/82ec18] Submitted process > NFCORE_RNASEQ:RNASEQ:QUANTIFY_SALMON:SALMON_SE_TRANSCRIPT (salmon_tx2gene.tsv)\n", "[64/b7326e] Submitted process > NFCORE_RNASEQ:RNASEQ:DESEQ2_QC_SALMON\n", "[9d/f22db1] Submitted process > NFCORE_RNASEQ:RNASEQ:QUANTIFY_SALMON:SALMON_SE_GENE_LENGTH_SCALED (salmon_tx2gene.tsv)\n", "[4f/ab9414] Submitted process > NFCORE_RNASEQ:RNASEQ:QUANTIFY_SALMON:SALMON_SE_GENE_SCALED (salmon_tx2gene.tsv)\n", "[7c/03149d] Submitted process > NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:BAM_SORT_SAMTOOLS:SAMTOOLS_SORT (WT_REP1)\n", "[c9/a7dcbc] Submitted process > NFCORE_RNASEQ:RNASEQ:QUANTIFY_STAR_SALMON:SALMON_QUANT (WT_REP1)\n", "[af/f84c6b] Submitted process > NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:BAM_SORT_SAMTOOLS:SAMTOOLS_INDEX (WT_REP1)\n", "[28/f82c0f] Submitted process > NFCORE_RNASEQ:RNASEQ:MARK_DUPLICATES_PICARD:PICARD_MARKDUPLICATES (WT_REP1)\n", "[ec/42ea5a] Submitted process > NFCORE_RNASEQ:RNASEQ:PRESEQ_LCEXTRAP (WT_REP1)\n", "[5b/0a7c7f] Submitted process > NFCORE_RNASEQ:RNASEQ:QUANTIFY_STAR_SALMON:SALMON_TX2GENE (genome_gfp.gtf)\n", "Pulling Singularity image https://depot.galaxyproject.org/singularity/rseqc:3.0.1--py37h516909a_1 [cache /home/jillian/.singularity/depot.galaxyproject.org-singularity-rseqc-3.0.1--py37h516909a_1.img]\n", "Pulling Singularity image https://depot.galaxyproject.org/singularity/qualimap:2.2.2d--1 [cache /home/jillian/.singularity/depot.galaxyproject.org-singularity-qualimap-2.2.2d--1.img]\n", "Pulling Singularity image https://depot.galaxyproject.org/singularity/bioconductor-dupradar:1.18.0--r40_1 [cache /home/jillian/.singularity/depot.galaxyproject.org-singularity-bioconductor-dupradar-1.18.0--r40_1.img]\n", "[96/f849a9] Submitted process > NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:BAM_SORT_SAMTOOLS:BAM_STATS_SAMTOOLS:SAMTOOLS_IDXSTATS (WT_REP1)\n", "[03/adcaf1] Submitted process > NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:BAM_SORT_SAMTOOLS:BAM_STATS_SAMTOOLS:SAMTOOLS_FLAGSTAT (WT_REP1)\n", "[01/e03f95] Submitted process > NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:BAM_SORT_SAMTOOLS:BAM_STATS_SAMTOOLS:SAMTOOLS_STATS (WT_REP1)\n", "[57/25a92f] Submitted process > NFCORE_RNASEQ:RNASEQ:STRINGTIE_STRINGTIE (WT_REP1)\n", "[59/f74ce8] Submitted process > NFCORE_RNASEQ:RNASEQ:MARK_DUPLICATES_PICARD:SAMTOOLS_INDEX (WT_REP1)\n", "[9c/a4fc4c] Submitted process > NFCORE_RNASEQ:RNASEQ:SUBREAD_FEATURECOUNTS (WT_REP1)\n", "[e4/98f601] Submitted process > NFCORE_RNASEQ:RNASEQ:BEDTOOLS_GENOMECOV (WT_REP1)\n", "[90/87a7e7] Submitted process > NFCORE_RNASEQ:RNASEQ:QUANTIFY_STAR_SALMON:SALMON_TXIMPORT\n", "[80/8dfb7e] Submitted process > NFCORE_RNASEQ:RNASEQ:RSEQC:RSEQC_READDISTRIBUTION (WT_REP1)\n", "[55/af088f] Submitted process > NFCORE_RNASEQ:RNASEQ:RSEQC:RSEQC_READDUPLICATION (WT_REP1)\n", "[95/1d62ea] Submitted process > NFCORE_RNASEQ:RNASEQ:RSEQC:RSEQC_BAMSTAT (WT_REP1)\n", "[1c/ca8a49] Submitted process > NFCORE_RNASEQ:RNASEQ:RSEQC:RSEQC_JUNCTIONANNOTATION (WT_REP1)\n", "[cd/f586ef] Submitted process > NFCORE_RNASEQ:RNASEQ:RSEQC:RSEQC_JUNCTIONSATURATION (WT_REP1)\n", "[1e/2adbe7] Submitted process > NFCORE_RNASEQ:RNASEQ:RSEQC:RSEQC_INNERDISTANCE (WT_REP1)\n", "[b3/0b2b07] Submitted process > NFCORE_RNASEQ:RNASEQ:RSEQC:RSEQC_INFEREXPERIMENT (WT_REP1)\n", "[de/194729] Submitted process > NFCORE_RNASEQ:RNASEQ:DUPRADAR (WT_REP1)\n", "Pulling Singularity image https://depot.galaxyproject.org/singularity/ucsc-bedclip:377--h0b8a92a_2 [cache /home/jillian/.singularity/depot.galaxyproject.org-singularity-ucsc-bedclip-377--h0b8a92a_2.img]\n", "[7d/4f0a0f] Submitted process > NFCORE_RNASEQ:RNASEQ:MARK_DUPLICATES_PICARD:BAM_STATS_SAMTOOLS:SAMTOOLS_FLAGSTAT (WT_REP1)\n", "[be/021104] Submitted process > NFCORE_RNASEQ:RNASEQ:MARK_DUPLICATES_PICARD:BAM_STATS_SAMTOOLS:SAMTOOLS_STATS (WT_REP1)\n", "[12/c9c076] Submitted process > NFCORE_RNASEQ:RNASEQ:MARK_DUPLICATES_PICARD:BAM_STATS_SAMTOOLS:SAMTOOLS_IDXSTATS (WT_REP1)\n", "[1e/cab3b2] Submitted process > NFCORE_RNASEQ:RNASEQ:MULTIQC_CUSTOM_BIOTYPE (WT_REP1)\n", "[82/d49de1] Submitted process > NFCORE_RNASEQ:RNASEQ:BEDGRAPH_TO_BIGWIG_REVERSE:UCSC_BEDCLIP (WT_REP1)\n", "[f2/7fa635] Submitted process > NFCORE_RNASEQ:RNASEQ:BEDGRAPH_TO_BIGWIG_FORWARD:UCSC_BEDCLIP (WT_REP1)\n", "[4a/888e88] Submitted process > NFCORE_RNASEQ:RNASEQ:QUALIMAP_RNASEQ (WT_REP1)\n", "[f5/d56240] Submitted process > NFCORE_RNASEQ:RNASEQ:QUANTIFY_STAR_SALMON:SALMON_SE_GENE_SCALED (salmon_tx2gene.tsv)\n", "[fb/235340] Submitted process > NFCORE_RNASEQ:RNASEQ:QUANTIFY_STAR_SALMON:SALMON_SE_TRANSCRIPT (salmon_tx2gene.tsv)\n", "[7a/d270dc] Submitted process > NFCORE_RNASEQ:RNASEQ:DESEQ2_QC_STAR_SALMON\n", "[9e/41609d] Submitted process > NFCORE_RNASEQ:RNASEQ:QUANTIFY_STAR_SALMON:SALMON_SE_GENE_LENGTH_SCALED (salmon_tx2gene.tsv)\n", "[b9/c49a83] Submitted process > NFCORE_RNASEQ:RNASEQ:QUANTIFY_STAR_SALMON:SALMON_SE_GENE (salmon_tx2gene.tsv)\n", "Pulling Singularity image https://depot.galaxyproject.org/singularity/ucsc-bedgraphtobigwig:377--h446ed27_1 [cache /home/jillian/.singularity/depot.galaxyproject.org-singularity-ucsc-bedgraphtobigwig-377--h446ed27_1.img]\n", "[a2/99b884] Submitted process > NFCORE_RNASEQ:RNASEQ:BEDGRAPH_TO_BIGWIG_REVERSE:UCSC_BEDGRAPHTOBIGWIG (WT_REP1)\n", "[a6/39fca2] Submitted process > NFCORE_RNASEQ:RNASEQ:BEDGRAPH_TO_BIGWIG_FORWARD:UCSC_BEDGRAPHTOBIGWIG (WT_REP1)\n", "Pulling Singularity image https://depot.galaxyproject.org/singularity/multiqc:1.13--pyhdfd78af_0 [cache /home/jillian/.singularity/depot.galaxyproject.org-singularity-multiqc-1.13--pyhdfd78af_0.img]\n", "[b3/f5b770] Submitted process > NFCORE_RNASEQ:RNASEQ:CUSTOM_DUMPSOFTWAREVERSIONS (1)\n", "[1a/dfc233] Submitted process > NFCORE_RNASEQ:RNASEQ:MULTIQC (1)\n", "-\u001b[0;35m[nf-core/rnaseq]\u001b[0;32m 1/1 samples passed STAR 5% mapped threshold:\n", " 87.67%: WT_REP1\n", "\u001b[0m-\n", "-\u001b[0;35m[nf-core/rnaseq]\u001b[0;32m Pipeline completed successfully\u001b[0m-\n", "exit\n", "Restarting Bash" ] } ], "source": [ "export NXF_ANSI_LOG=\"False\"\n", "export NXF_CONDA_CACHEDIR=\"${HOME}/.conda/envs\"\n", "export NXF_SINGULARITY_CACHEDIR=\"${HOME}/.singularity\"\n", "\n", "module load nextflow\n", "module load singularity-3.8.5-gcc-7.3.1-g2xhg2m\n", "\n", "nextflow \\\n", " run \\\n", " nf-core/rnaseq \\\n", " -with-trace \\\n", " -with-report \\\n", " -with-dag \\\n", " -w ./work \\\n", " --input ./samplesheet_test.csv \\\n", " -resume \\\n", " -profile slurm \\\n", " -c test.config \\\n", " -c slurm-default.config \\\n", " -c ~/slurm.config \\\n", " --outdir ./results \n", "\n", "exit 0" ] }, { "cell_type": "code", "execution_count": 14, "id": "bdb0a28f-9cf6-446a-bc3c-d9ba86936ad3", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "results\n", "├── bbmap\n", "│   ├── bbsplit\n", "│   │   └── ref\n", "│   │   ├── genome\n", "│   │   │   └── 1\n", "│   │   │   ├── chr1.chrom.gz\n", "│   │   │   ├── info.txt\n", "│   │   │   ├── merged_ref_8374379829187813017.fa.gz\n", "│   │   │   ├── namelist.txt\n", "│   │   │   ├── reflist.txt\n", "│   │   │   ├── scaffolds.txt.gz\n", "│   │   │   └── summary.txt\n", "│   │   └── index\n", "│   │   └── 1\n", "│   │   ├── chr1_index_k13_c12_b1.block\n", "│   │   └── chr1_index_k13_c12_b1.block2.gz\n", "│   ├── WT_REP1_human_1.fastq.gz\n", "│   ├── WT_REP1_human_2.fastq.gz\n", "│   ├── WT_REP1_primary_1.fastq.gz\n", "│   ├── WT_REP1_primary_2.fastq.gz\n", "│   ├── WT_REP1_sarscov2_1.fastq.gz\n", "│   ├── WT_REP1_sarscov2_2.fastq.gz\n", "│   └── WT_REP1.stats.txt\n", "├── deseq2\n", "│   └── deseq2.dds.RData\n", "├── fastqc\n", "│   ├── WT_REP1_1_fastqc.html\n", "│   ├── WT_REP1_1_fastqc.zip\n", "│   ├── WT_REP1_2_fastqc.html\n", "│   └── WT_REP1_2_fastqc.zip\n", "├── multiqc\n", "│   └── star_salmon\n", "│   ├── multiqc_data\n", "│   │   ├── junction_saturation_known.txt\n", "│   │   ├── junction_saturation_novel.txt\n", "│   │   ├── mqc_cutadapt_filtered_reads_plot_1.txt\n", "│   │   ├── mqc_cutadapt_trimmed_sequences_plot_3_Counts.txt\n", "│   │   ├── mqc_cutadapt_trimmed_sequences_plot_3_Obs_Exp.txt\n", "│   │   ├── mqc_fastqc_adapter_content_plot_1.txt\n", "│   │   ├── mqc_fastqc_overrepresented_sequences_plot_1.txt\n", "│   │   ├── mqc_fastqc_overrepresented_sequences_plot-2_1.txt\n", "│   │   ├── mqc_fastqc_per_base_n_content_plot_1.txt\n", "│   │   ├── mqc_fastqc_per_base_n_content_plot-2_1.txt\n", "│   │   ├── mqc_fastqc_per_base_sequence_quality_plot_1.txt\n", "│   │   ├── mqc_fastqc_per_base_sequence_quality_plot-2_1.txt\n", "│   │   ├── mqc_fastqc_per_sequence_gc_content_plot-2_Counts.txt\n", "│   │   ├── mqc_fastqc_per_sequence_gc_content_plot-2_Percentages.txt\n", "│   │   ├── mqc_fastqc_per_sequence_gc_content_plot_Counts.txt\n", "│   │   ├── mqc_fastqc_per_sequence_gc_content_plot_Percentages.txt\n", "│   │   ├── mqc_fastqc_per_sequence_quality_scores_plot_1.txt\n", "│   │   ├── mqc_fastqc_per_sequence_quality_scores_plot-2_1.txt\n", "│   │   ├── mqc_fastqc_sequence_counts_plot_1.txt\n", "│   │   ├── mqc_fastqc_sequence_counts_plot-2_1.txt\n", "│   │   ├── mqc_fastqc_sequence_duplication_levels_plot_1.txt\n", "│   │   ├── mqc_fastqc_sequence_duplication_levels_plot-2_1.txt\n", "│   │   ├── mqc_fastqc_sequence_length_distribution_plot_1.txt\n", "│   │   ├── mqc_picard_deduplication_1.txt\n", "│   │   ├── mqc_preseq_plot_1.txt\n", "│   │   ├── mqc_qualimap_gene_coverage_profile_Counts.txt\n", "│   │   ├── mqc_qualimap_gene_coverage_profile_Normalised.txt\n", "│   │   ├── mqc_qualimap_genomic_origin_1.txt\n", "│   │   ├── mqc_rseqc_infer_experiment_plot_1.txt\n", "│   │   ├── mqc_rseqc_inner_distance_plot_Counts.txt\n", "│   │   ├── mqc_rseqc_inner_distance_plot_Percentages.txt\n", "│   │   ├── mqc_rseqc_junction_annotation_junctions_plot_Events.txt\n", "│   │   ├── mqc_rseqc_junction_annotation_junctions_plot_Junctions.txt\n", "│   │   ├── mqc_rseqc_junction_saturation_plot_All_Junctions.txt\n", "│   │   ├── mqc_rseqc_junction_saturation_plot_Known_Junctions.txt\n", "│   │   ├── mqc_rseqc_junction_saturation_plot_Novel_Junctions.txt\n", "│   │   ├── mqc_rseqc_read_distribution_plot_1.txt\n", "│   │   ├── mqc_rseqc_read_dups_plot_1.txt\n", "│   │   ├── mqc_salmon_plot_1.txt\n", "│   │   ├── mqc_samtools_alignment_plot_1.txt\n", "│   │   ├── mqc_samtools-idxstats-mapped-reads-plot_Normalised_Counts.txt\n", "│   │   ├── mqc_samtools-idxstats-mapped-reads-plot_Observed_over_Expected_Counts.txt\n", "│   │   ├── mqc_samtools-idxstats-mapped-reads-plot_Raw_Counts.txt\n", "│   │   ├── mqc_star_alignment_plot_1.txt\n", "│   │   ├── multiqc_citations.txt\n", "│   │   ├── multiqc_cutadapt.txt\n", "│   │   ├── multiqc_data.json\n", "│   │   ├── multiqc_dupradar-plot.txt\n", "│   │   ├── multiqc_fastqc_1.txt\n", "│   │   ├── multiqc_fastqc.txt\n", "│   │   ├── multiqc_featurecounts_biotype_plot.txt\n", "│   │   ├── multiqc_general_stats.txt\n", "│   │   ├── multiqc.log\n", "│   │   ├── multiqc_picard_dups.txt\n", "│   │   ├── multiqc_rseqc_bam_stat.txt\n", "│   │   ├── multiqc_rseqc_infer_experiment.txt\n", "│   │   ├── multiqc_rseqc_junction_annotation.txt\n", "│   │   ├── multiqc_rseqc_read_distribution.txt\n", "│   │   ├── multiqc_salmon.txt\n", "│   │   ├── multiqc_samtools_flagstat.txt\n", "│   │   ├── multiqc_samtools_idxstats.txt\n", "│   │   ├── multiqc_samtools_stats.txt\n", "│   │   ├── multiqc_sources.txt\n", "│   │   ├── multiqc_star.txt\n", "│   │   ├── picard_histogram_1.txt\n", "│   │   ├── picard_histogram_2.txt\n", "│   │   ├── picard_histogram.txt\n", "│   │   ├── preseq.txt\n", "│   │   ├── qualimap_rnaseq_cov_hist.txt\n", "│   │   ├── qualimap_rnaseq_genome_results.txt\n", "│   │   ├── rseqc_inner_distance.txt\n", "│   │   ├── rseqc_junction_saturation_all.txt\n", "│   │   └── rseqc_read_dups.txt\n", "│   ├── multiqc_plots\n", "│   │   ├── pdf\n", "│   │   │   ├── mqc_cutadapt_filtered_reads_plot_1_pc.pdf\n", "│   │   │   ├── mqc_cutadapt_filtered_reads_plot_1.pdf\n", "│   │   │   ├── mqc_cutadapt_trimmed_sequences_plot_3_Counts.pdf\n", "│   │   │   ├── mqc_cutadapt_trimmed_sequences_plot_3_Obs_Exp.pdf\n", "│   │   │   ├── mqc_dupradar-plot_1.pdf\n", "│   │   │   ├── mqc_fastqc_adapter_content_plot_1.pdf\n", "│   │   │   ├── mqc_fastqc_overrepresented_sequences_plot_1.pdf\n", "│   │   │   ├── mqc_fastqc_overrepresented_sequences_plot-2_1.pdf\n", "│   │   │   ├── mqc_fastqc_per_base_n_content_plot_1.pdf\n", "│   │   │   ├── mqc_fastqc_per_base_n_content_plot-2_1.pdf\n", "│   │   │   ├── mqc_fastqc_per_base_sequence_quality_plot_1.pdf\n", "│   │   │   ├── mqc_fastqc_per_base_sequence_quality_plot-2_1.pdf\n", "│   │   │   ├── mqc_fastqc_per_sequence_gc_content_plot-2_Counts.pdf\n", "│   │   │   ├── mqc_fastqc_per_sequence_gc_content_plot-2_Percentages.pdf\n", "│   │   │   ├── mqc_fastqc_per_sequence_gc_content_plot_Counts.pdf\n", "│   │   │   ├── mqc_fastqc_per_sequence_gc_content_plot_Percentages.pdf\n", "│   │   │   ├── mqc_fastqc_per_sequence_quality_scores_plot_1.pdf\n", "│   │   │   ├── mqc_fastqc_per_sequence_quality_scores_plot-2_1.pdf\n", "│   │   │   ├── mqc_fastqc_sequence_counts_plot_1_pc.pdf\n", "│   │   │   ├── mqc_fastqc_sequence_counts_plot_1.pdf\n", "│   │   │   ├── mqc_fastqc_sequence_counts_plot-2_1_pc.pdf\n", "│   │   │   ├── mqc_fastqc_sequence_counts_plot-2_1.pdf\n", "│   │   │   ├── mqc_fastqc_sequence_duplication_levels_plot_1.pdf\n", "│   │   │   ├── mqc_fastqc_sequence_duplication_levels_plot-2_1.pdf\n", "│   │   │   ├── mqc_fastqc_sequence_length_distribution_plot_1.pdf\n", "│   │   │   ├── mqc_featurecounts_biotype_plot_1_pc.pdf\n", "│   │   │   ├── mqc_featurecounts_biotype_plot_1.pdf\n", "│   │   │   ├── mqc_picard_deduplication_1_pc.pdf\n", "│   │   │   ├── mqc_picard_deduplication_1.pdf\n", "│   │   │   ├── mqc_preseq_plot_1.pdf\n", "│   │   │   ├── mqc_qualimap_gene_coverage_profile_Counts.pdf\n", "│   │   │   ├── mqc_qualimap_gene_coverage_profile_Normalised.pdf\n", "│   │   │   ├── mqc_qualimap_genomic_origin_1_pc.pdf\n", "│   │   │   ├── mqc_qualimap_genomic_origin_1.pdf\n", "│   │   │   ├── mqc_rseqc_infer_experiment_plot_1.pdf\n", "│   │   │   ├── mqc_rseqc_inner_distance_plot_Counts.pdf\n", "│   │   │   ├── mqc_rseqc_inner_distance_plot_Percentages.pdf\n", "│   │   │   ├── mqc_rseqc_junction_annotation_junctions_plot_Events_pc.pdf\n", "│   │   │   ├── mqc_rseqc_junction_annotation_junctions_plot_Events.pdf\n", "│   │   │   ├── mqc_rseqc_junction_annotation_junctions_plot_Junctions_pc.pdf\n", "│   │   │   ├── mqc_rseqc_junction_annotation_junctions_plot_Junctions.pdf\n", "│   │   │   ├── mqc_rseqc_junction_saturation_plot_All_Junctions.pdf\n", "│   │   │   ├── mqc_rseqc_junction_saturation_plot_Known_Junctions.pdf\n", "│   │   │   ├── mqc_rseqc_junction_saturation_plot_Novel_Junctions.pdf\n", "│   │   │   ├── mqc_rseqc_read_distribution_plot_1_pc.pdf\n", "│   │   │   ├── mqc_rseqc_read_distribution_plot_1.pdf\n", "│   │   │   ├── mqc_rseqc_read_dups_plot_1.pdf\n", "│   │   │   ├── mqc_salmon_plot_1.pdf\n", "│   │   │   ├── mqc_samtools_alignment_plot_1_pc.pdf\n", "│   │   │   ├── mqc_samtools_alignment_plot_1.pdf\n", "│   │   │   ├── mqc_samtools-idxstats-mapped-reads-plot_Normalised_Counts.pdf\n", "│   │   │   ├── mqc_samtools-idxstats-mapped-reads-plot_Observed_over_Expected_Counts.pdf\n", "│   │   │   ├── mqc_samtools-idxstats-mapped-reads-plot_Raw_Counts.pdf\n", "│   │   │   ├── mqc_star_alignment_plot_1_pc.pdf\n", "│   │   │   └── mqc_star_alignment_plot_1.pdf\n", "│   │   ├── png\n", "│   │   │   ├── mqc_cutadapt_filtered_reads_plot_1_pc.png\n", "│   │   │   ├── mqc_cutadapt_filtered_reads_plot_1.png\n", "│   │   │   ├── mqc_cutadapt_trimmed_sequences_plot_3_Counts.png\n", "│   │   │   ├── mqc_cutadapt_trimmed_sequences_plot_3_Obs_Exp.png\n", "│   │   │   ├── mqc_dupradar-plot_1.png\n", "│   │   │   ├── mqc_fastqc_adapter_content_plot_1.png\n", "│   │   │   ├── mqc_fastqc_overrepresented_sequences_plot_1.png\n", "│   │   │   ├── mqc_fastqc_overrepresented_sequences_plot-2_1.png\n", "│   │   │   ├── mqc_fastqc_per_base_n_content_plot_1.png\n", "│   │   │   ├── mqc_fastqc_per_base_n_content_plot-2_1.png\n", "│   │   │   ├── mqc_fastqc_per_base_sequence_quality_plot_1.png\n", "│   │   │   ├── mqc_fastqc_per_base_sequence_quality_plot-2_1.png\n", "│   │   │   ├── mqc_fastqc_per_sequence_gc_content_plot-2_Counts.png\n", "│   │   │   ├── mqc_fastqc_per_sequence_gc_content_plot-2_Percentages.png\n", "│   │   │   ├── mqc_fastqc_per_sequence_gc_content_plot_Counts.png\n", "│   │   │   ├── mqc_fastqc_per_sequence_gc_content_plot_Percentages.png\n", "│   │   │   ├── mqc_fastqc_per_sequence_quality_scores_plot_1.png\n", "│   │   │   ├── mqc_fastqc_per_sequence_quality_scores_plot-2_1.png\n", "│   │   │   ├── mqc_fastqc_sequence_counts_plot_1_pc.png\n", "│   │   │   ├── mqc_fastqc_sequence_counts_plot_1.png\n", "│   │   │   ├── mqc_fastqc_sequence_counts_plot-2_1_pc.png\n", "│   │   │   ├── mqc_fastqc_sequence_counts_plot-2_1.png\n", "│   │   │   ├── mqc_fastqc_sequence_duplication_levels_plot_1.png\n", "│   │   │   ├── mqc_fastqc_sequence_duplication_levels_plot-2_1.png\n", "│   │   │   ├── mqc_fastqc_sequence_length_distribution_plot_1.png\n", "│   │   │   ├── mqc_featurecounts_biotype_plot_1_pc.png\n", "│   │   │   ├── mqc_featurecounts_biotype_plot_1.png\n", "│   │   │   ├── mqc_picard_deduplication_1_pc.png\n", "│   │   │   ├── mqc_picard_deduplication_1.png\n", "│   │   │   ├── mqc_preseq_plot_1.png\n", "│   │   │   ├── mqc_qualimap_gene_coverage_profile_Counts.png\n", "│   │   │   ├── mqc_qualimap_gene_coverage_profile_Normalised.png\n", "│   │   │   ├── mqc_qualimap_genomic_origin_1_pc.png\n", "│   │   │   ├── mqc_qualimap_genomic_origin_1.png\n", "│   │   │   ├── mqc_rseqc_infer_experiment_plot_1.png\n", "│   │   │   ├── mqc_rseqc_inner_distance_plot_Counts.png\n", "│   │   │   ├── mqc_rseqc_inner_distance_plot_Percentages.png\n", "│   │   │   ├── mqc_rseqc_junction_annotation_junctions_plot_Events_pc.png\n", "│   │   │   ├── mqc_rseqc_junction_annotation_junctions_plot_Events.png\n", "│   │   │   ├── mqc_rseqc_junction_annotation_junctions_plot_Junctions_pc.png\n", "│   │   │   ├── mqc_rseqc_junction_annotation_junctions_plot_Junctions.png\n", "│   │   │   ├── mqc_rseqc_junction_saturation_plot_All_Junctions.png\n", "│   │   │   ├── mqc_rseqc_junction_saturation_plot_Known_Junctions.png\n", "│   │   │   ├── mqc_rseqc_junction_saturation_plot_Novel_Junctions.png\n", "│   │   │   ├── mqc_rseqc_read_distribution_plot_1_pc.png\n", "│   │   │   ├── mqc_rseqc_read_distribution_plot_1.png\n", "│   │   │   ├── mqc_rseqc_read_dups_plot_1.png\n", "│   │   │   ├── mqc_salmon_plot_1.png\n", "│   │   │   ├── mqc_samtools_alignment_plot_1_pc.png\n", "│   │   │   ├── mqc_samtools_alignment_plot_1.png\n", "│   │   │   ├── mqc_samtools-idxstats-mapped-reads-plot_Normalised_Counts.png\n", "│   │   │   ├── mqc_samtools-idxstats-mapped-reads-plot_Observed_over_Expected_Counts.png\n", "│   │   │   ├── mqc_samtools-idxstats-mapped-reads-plot_Raw_Counts.png\n", "│   │   │   ├── mqc_star_alignment_plot_1_pc.png\n", "│   │   │   └── mqc_star_alignment_plot_1.png\n", "│   │   └── svg\n", "│   │   ├── mqc_cutadapt_filtered_reads_plot_1_pc.svg\n", "│   │   ├── mqc_cutadapt_filtered_reads_plot_1.svg\n", "│   │   ├── mqc_cutadapt_trimmed_sequences_plot_3_Counts.svg\n", "│   │   ├── mqc_cutadapt_trimmed_sequences_plot_3_Obs_Exp.svg\n", "│   │   ├── mqc_dupradar-plot_1.svg\n", "│   │   ├── mqc_fastqc_adapter_content_plot_1.svg\n", "│   │   ├── mqc_fastqc_overrepresented_sequences_plot_1.svg\n", "│   │   ├── mqc_fastqc_overrepresented_sequences_plot-2_1.svg\n", "│   │   ├── mqc_fastqc_per_base_n_content_plot_1.svg\n", "│   │   ├── mqc_fastqc_per_base_n_content_plot-2_1.svg\n", "│   │   ├── mqc_fastqc_per_base_sequence_quality_plot_1.svg\n", "│   │   ├── mqc_fastqc_per_base_sequence_quality_plot-2_1.svg\n", "│   │   ├── mqc_fastqc_per_sequence_gc_content_plot-2_Counts.svg\n", "│   │   ├── mqc_fastqc_per_sequence_gc_content_plot-2_Percentages.svg\n", "│   │   ├── mqc_fastqc_per_sequence_gc_content_plot_Counts.svg\n", "│   │   ├── mqc_fastqc_per_sequence_gc_content_plot_Percentages.svg\n", "│   │   ├── mqc_fastqc_per_sequence_quality_scores_plot_1.svg\n", "│   │   ├── mqc_fastqc_per_sequence_quality_scores_plot-2_1.svg\n", "│   │   ├── mqc_fastqc_sequence_counts_plot_1_pc.svg\n", "│   │   ├── mqc_fastqc_sequence_counts_plot_1.svg\n", "│   │   ├── mqc_fastqc_sequence_counts_plot-2_1_pc.svg\n", "│   │   ├── mqc_fastqc_sequence_counts_plot-2_1.svg\n", "│   │   ├── mqc_fastqc_sequence_duplication_levels_plot_1.svg\n", "│   │   ├── mqc_fastqc_sequence_duplication_levels_plot-2_1.svg\n", "│   │   ├── mqc_fastqc_sequence_length_distribution_plot_1.svg\n", "│   │   ├── mqc_featurecounts_biotype_plot_1_pc.svg\n", "│   │   ├── mqc_featurecounts_biotype_plot_1.svg\n", "│   │   ├── mqc_picard_deduplication_1_pc.svg\n", "│   │   ├── mqc_picard_deduplication_1.svg\n", "│   │   ├── mqc_preseq_plot_1.svg\n", "│   │   ├── mqc_qualimap_gene_coverage_profile_Counts.svg\n", "│   │   ├── mqc_qualimap_gene_coverage_profile_Normalised.svg\n", "│   │   ├── mqc_qualimap_genomic_origin_1_pc.svg\n", "│   │   ├── mqc_qualimap_genomic_origin_1.svg\n", "│   │   ├── mqc_rseqc_infer_experiment_plot_1.svg\n", "│   │   ├── mqc_rseqc_inner_distance_plot_Counts.svg\n", "│   │   ├── mqc_rseqc_inner_distance_plot_Percentages.svg\n", "│   │   ├── mqc_rseqc_junction_annotation_junctions_plot_Events_pc.svg\n", "│   │   ├── mqc_rseqc_junction_annotation_junctions_plot_Events.svg\n", "│   │   ├── mqc_rseqc_junction_annotation_junctions_plot_Junctions_pc.svg\n", "│   │   ├── mqc_rseqc_junction_annotation_junctions_plot_Junctions.svg\n", "│   │   ├── mqc_rseqc_junction_saturation_plot_All_Junctions.svg\n", "│   │   ├── mqc_rseqc_junction_saturation_plot_Known_Junctions.svg\n", "│   │   ├── mqc_rseqc_junction_saturation_plot_Novel_Junctions.svg\n", "│   │   ├── mqc_rseqc_read_distribution_plot_1_pc.svg\n", "│   │   ├── mqc_rseqc_read_distribution_plot_1.svg\n", "│   │   ├── mqc_rseqc_read_dups_plot_1.svg\n", "│   │   ├── mqc_salmon_plot_1.svg\n", "│   │   ├── mqc_samtools_alignment_plot_1_pc.svg\n", "│   │   ├── mqc_samtools_alignment_plot_1.svg\n", "│   │   ├── mqc_samtools-idxstats-mapped-reads-plot_Normalised_Counts.svg\n", "│   │   ├── mqc_samtools-idxstats-mapped-reads-plot_Observed_over_Expected_Counts.svg\n", "│   │   ├── mqc_samtools-idxstats-mapped-reads-plot_Raw_Counts.svg\n", "│   │   ├── mqc_star_alignment_plot_1_pc.svg\n", "│   │   └── mqc_star_alignment_plot_1.svg\n", "│   └── multiqc_report.html\n", "├── pipeline_info\n", "│   ├── execution_report_2022-11-28_08-50-33.html\n", "│   ├── execution_report_2022-11-28_08-55-06.html\n", "│   ├── execution_timeline_2022-11-28_08-50-33.html\n", "│   ├── execution_timeline_2022-11-28_08-55-06.html\n", "│   ├── execution_trace_2022-11-28_08-55-06.txt\n", "│   ├── execution_trace_2022-11-28_09-00-31.txt\n", "│   ├── pipeline_dag_2022-11-28_08-50-33.html\n", "│   ├── pipeline_dag_2022-11-28_08-55-06.html\n", "│   ├── samplesheet.valid.csv\n", "│   └── software_versions.yml\n", "├── salmon\n", "│   ├── salmon.merged.gene_counts_length_scaled.rds\n", "│   ├── salmon.merged.gene_counts_length_scaled.tsv\n", "│   ├── salmon.merged.gene_counts.rds\n", "│   ├── salmon.merged.gene_counts_scaled.rds\n", "│   ├── salmon.merged.gene_counts_scaled.tsv\n", "│   ├── salmon.merged.gene_counts.tsv\n", "│   ├── salmon.merged.gene_tpm.tsv\n", "│   ├── salmon.merged.transcript_counts.rds\n", "│   ├── salmon.merged.transcript_counts.tsv\n", "│   ├── salmon.merged.transcript_tpm.tsv\n", "│   ├── salmon_tx2gene.tsv\n", "│   └── WT_REP1\n", "│   ├── aux_info\n", "│   │   ├── ambig_info.tsv\n", "│   │   ├── expected_bias.gz\n", "│   │   ├── fld.gz\n", "│   │   ├── meta_info.json\n", "│   │   ├── observed_bias_3p.gz\n", "│   │   └── observed_bias.gz\n", "│   ├── cmd_info.json\n", "│   ├── lib_format_counts.json\n", "│   ├── libParams\n", "│   │   └── flenDist.txt\n", "│   ├── logs\n", "│   │   └── salmon_quant.log\n", "│   ├── quant.genes.sf\n", "│   └── quant.sf\n", "├── star_salmon\n", "│   ├── bigwig\n", "│   │   ├── WT_REP1.forward.bigWig\n", "│   │   └── WT_REP1.reverse.bigWig\n", "│   ├── deseq2_qc\n", "│   │   └── deseq2.dds.RData\n", "│   ├── dupradar\n", "│   │   ├── box_plot\n", "│   │   │   └── WT_REP1_duprateExpBoxplot.pdf\n", "│   │   ├── gene_data\n", "│   │   │   └── WT_REP1_dupMatrix.txt\n", "│   │   ├── histogram\n", "│   │   │   └── WT_REP1_expressionHist.pdf\n", "│   │   ├── intercepts_slope\n", "│   │   │   └── WT_REP1_intercept_slope.txt\n", "│   │   └── scatter_plot\n", "│   │   └── WT_REP1_duprateExpDens.pdf\n", "│   ├── featurecounts\n", "│   │   ├── WT_REP1.biotype_counts_mqc.tsv\n", "│   │   ├── WT_REP1.biotype_counts_rrna_mqc.tsv\n", "│   │   ├── WT_REP1.featureCounts.txt\n", "│   │   └── WT_REP1.featureCounts.txt.summary\n", "│   ├── log\n", "│   │   ├── WT_REP1.Log.final.out\n", "│   │   ├── WT_REP1.Log.out\n", "│   │   ├── WT_REP1.Log.progress.out\n", "│   │   └── WT_REP1.SJ.out.tab\n", "│   ├── picard_metrics\n", "│   │   └── WT_REP1.markdup.sorted.MarkDuplicates.metrics.txt\n", "│   ├── preseq\n", "│   │   ├── log\n", "│   │   │   └── WT_REP1.command.log\n", "│   │   └── WT_REP1.lc_extrap.txt\n", "│   ├── qualimap\n", "│   │   └── WT_REP1\n", "│   │   ├── css\n", "│   │   │   ├── agogo.css\n", "│   │   │   ├── ajax-loader.gif\n", "│   │   │   ├── basic.css\n", "│   │   │   ├── bgfooter.png\n", "│   │   │   ├── bgtop.png\n", "│   │   │   ├── comment-bright.png\n", "│   │   │   ├── comment-close.png\n", "│   │   │   ├── comment.png\n", "│   │   │   ├── doctools.js\n", "│   │   │   ├── down.png\n", "│   │   │   ├── down-pressed.png\n", "│   │   │   ├── file.png\n", "│   │   │   ├── jquery.js\n", "│   │   │   ├── minus.png\n", "│   │   │   ├── plus.png\n", "│   │   │   ├── pygments.css\n", "│   │   │   ├── qualimap_logo_small.png\n", "│   │   │   ├── report.css\n", "│   │   │   ├── searchtools.js\n", "│   │   │   ├── underscore.js\n", "│   │   │   ├── up.png\n", "│   │   │   ├── up-pressed.png\n", "│   │   │   └── websupport.js\n", "│   │   ├── images_qualimapReport\n", "│   │   │   ├── Coverage\\ Profile\\ Along\\ Genes\\ (High).png\n", "│   │   │   ├── Coverage\\ Profile\\ Along\\ Genes\\ (Low).png\n", "│   │   │   ├── Coverage\\ Profile\\ Along\\ Genes\\ (Total).png\n", "│   │   │   ├── Junction\\ Analysis.png\n", "│   │   │   ├── Reads\\ Genomic\\ Origin.png\n", "│   │   │   └── Transcript\\ coverage\\ histogram.png\n", "│   │   ├── qualimapReport.html\n", "│   │   ├── raw_data_qualimapReport\n", "│   │   │   ├── coverage_profile_along_genes_(high).txt\n", "│   │   │   ├── coverage_profile_along_genes_(low).txt\n", "│   │   │   └── coverage_profile_along_genes_(total).txt\n", "│   │   └── rnaseq_qc_results.txt\n", "│   ├── rseqc\n", "│   │   ├── bam_stat\n", "│   │   │   └── WT_REP1.bam_stat.txt\n", "│   │   ├── infer_experiment\n", "│   │   │   └── WT_REP1.infer_experiment.txt\n", "│   │   ├── inner_distance\n", "│   │   │   ├── pdf\n", "│   │   │   │   └── WT_REP1.inner_distance_plot.pdf\n", "│   │   │   ├── rscript\n", "│   │   │   │   └── WT_REP1.inner_distance_plot.r\n", "│   │   │   └── txt\n", "│   │   │   ├── WT_REP1.inner_distance_freq.txt\n", "│   │   │   ├── WT_REP1.inner_distance_mean.txt\n", "│   │   │   └── WT_REP1.inner_distance.txt\n", "│   │   ├── junction_annotation\n", "│   │   │   ├── bed\n", "│   │   │   │   ├── WT_REP1.junction.bed\n", "│   │   │   │   └── WT_REP1.junction.Interact.bed\n", "│   │   │   ├── log\n", "│   │   │   │   └── WT_REP1.junction_annotation.log\n", "│   │   │   ├── pdf\n", "│   │   │   │   ├── WT_REP1.splice_events.pdf\n", "│   │   │   │   └── WT_REP1.splice_junction.pdf\n", "│   │   │   ├── rscript\n", "│   │   │   │   └── WT_REP1.junction_plot.r\n", "│   │   │   └── xls\n", "│   │   │   └── WT_REP1.junction.xls\n", "│   │   ├── junction_saturation\n", "│   │   │   ├── pdf\n", "│   │   │   │   └── WT_REP1.junctionSaturation_plot.pdf\n", "│   │   │   └── rscript\n", "│   │   │   └── WT_REP1.junctionSaturation_plot.r\n", "│   │   ├── read_distribution\n", "│   │   │   └── WT_REP1.read_distribution.txt\n", "│   │   └── read_duplication\n", "│   │   ├── pdf\n", "│   │   │   └── WT_REP1.DupRate_plot.pdf\n", "│   │   ├── rscript\n", "│   │   │   └── WT_REP1.DupRate_plot.r\n", "│   │   └── xls\n", "│   │   ├── WT_REP1.pos.DupRate.xls\n", "│   │   └── WT_REP1.seq.DupRate.xls\n", "│   ├── salmon.merged.gene_counts_length_scaled.rds\n", "│   ├── salmon.merged.gene_counts_length_scaled.tsv\n", "│   ├── salmon.merged.gene_counts.rds\n", "│   ├── salmon.merged.gene_counts_scaled.rds\n", "│   ├── salmon.merged.gene_counts_scaled.tsv\n", "│   ├── salmon.merged.gene_counts.tsv\n", "│   ├── salmon.merged.gene_tpm.tsv\n", "│   ├── salmon.merged.transcript_counts.rds\n", "│   ├── salmon.merged.transcript_counts.tsv\n", "│   ├── salmon.merged.transcript_tpm.tsv\n", "│   ├── salmon_tx2gene.tsv\n", "│   ├── samtools_stats\n", "│   │   ├── WT_REP1.markdup.sorted.bam.flagstat\n", "│   │   ├── WT_REP1.markdup.sorted.bam.idxstats\n", "│   │   ├── WT_REP1.markdup.sorted.bam.stats\n", "│   │   ├── WT_REP1.sorted.bam.flagstat\n", "│   │   ├── WT_REP1.sorted.bam.idxstats\n", "│   │   └── WT_REP1.sorted.bam.stats\n", "│   ├── stringtie\n", "│   │   ├── WT_REP1.ballgown\n", "│   │   │   ├── e2t.ctab\n", "│   │   │   ├── e_data.ctab\n", "│   │   │   ├── i2t.ctab\n", "│   │   │   ├── i_data.ctab\n", "│   │   │   └── t_data.ctab\n", "│   │   ├── WT_REP1.coverage.gtf\n", "│   │   ├── WT_REP1.gene.abundance.txt\n", "│   │   └── WT_REP1.transcripts.gtf\n", "│   ├── WT_REP1\n", "│   │   ├── aux_info\n", "│   │   │   ├── ambig_info.tsv\n", "│   │   │   ├── expected_bias.gz\n", "│   │   │   ├── fld.gz\n", "│   │   │   ├── meta_info.json\n", "│   │   │   ├── observed_bias_3p.gz\n", "│   │   │   └── observed_bias.gz\n", "│   │   ├── cmd_info.json\n", "│   │   ├── libParams\n", "│   │   │   └── flenDist.txt\n", "│   │   ├── logs\n", "│   │   │   └── salmon_quant.log\n", "│   │   ├── quant.genes.sf\n", "│   │   └── quant.sf\n", "│   ├── WT_REP1.markdup.sorted.bam\n", "│   └── WT_REP1.markdup.sorted.bam.bai\n", "└── trimgalore\n", " ├── fastqc\n", " │   ├── WT_REP1_1_val_1_fastqc.html\n", " │   ├── WT_REP1_1_val_1_fastqc.zip\n", " │   ├── WT_REP1_2_val_2_fastqc.html\n", " │   └── WT_REP1_2_val_2_fastqc.zip\n", " ├── WT_REP1_1.fastq.gz_trimming_report.txt\n", " └── WT_REP1_2.fastq.gz_trimming_report.txt\n", "\n", "71 directories, 411 files\n" ] } ], "source": [ "tree results" ] }, { "cell_type": "code", "execution_count": 15, "id": "63166e50-1929-4d9d-a63b-4407d9e15892", "metadata": { "pycharm": { "name": "#%%\n" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "rm: cannot remove ‘.nextflow/cache/9ac749c4-d518-411e-906f-d39380596ec7/db’: Directory not empty\n", "rm: cannot remove ‘.nextflow/cache/9ac749c4-d518-411e-906f-d39380596ec7/db/.nfs000000004fc0228a0000000c’: Device or resource busy\n", "rm: cannot remove ‘.nextflow/cache/9ac749c4-d518-411e-906f-d39380596ec7/db/.nfs000000004fc0228d0000000d’: Device or resource busy\n", "rm: cannot remove ‘.nextflow/cache/9ac749c4-d518-411e-906f-d39380596ec7/db/.nfs000000004fc0228e0000000e’: Device or resource busy\n", "rm: cannot remove ‘.nextflow/cache/9ac749c4-d518-411e-906f-d39380596ec7/.nfs000000004f8009ea0000000f’: Device or resource busy\n" ] }, { "ename": "", "evalue": "1", "output_type": "error", "traceback": [] } ], "source": [ "# cleanup\n", "rm -rf .nextflow\n", "#sleep 1m\n", "#rm -rf .nextflow*" ] } ], "metadata": { "kernelspec": { "display_name": "Bash", "language": "bash", "name": "bash" }, "language_info": { "codemirror_mode": "shell", "file_extension": ".sh", "mimetype": "text/x-sh", "name": "bash" } }, "nbformat": 4, "nbformat_minor": 5 }