基于亚马逊云科技 CI/CD 和 Amazon Batch 服务构建云端单细胞分析流程

backend {
  default = "AWSBATCH"
  providers {
    AWSBATCH {
      actor-factory = "cromwell.backend.impl.aws.AwsBatchBackendLifecycleActorFactory"
      config {
        numSubmitAttempts = 10
        numCreateDefinitionAttempts = 10
        root = "s3://singlecellanalysis"
        auth = "default"
        default-runtime-attributes { queueArn = "arn:aws-cn:batch:cn-northwest-1:632427418419:job-queue/default-singlecellanalysis" , scriptBucketName = "singlecellanalysis" }
        filesystems {
          s3 {
            auth = "default"
            duplication-strategy: [
              "hard-link", "soft-link", "copy"
            ]
          }
        }
      }
    }
  }
}

FROM continuumio/miniconda3:4.9.2
ADD cellranger.tar.gz /opt/
RUN mv "$(find /opt -maxdepth 1 | grep cellranger)" "/opt/cellranger"

FROM continuumio/miniconda3:latest
COPY --from=0 /opt/cellranger /opt/cellranger
COPY src /opt/10x
ENV PATH="/opt/cellranger:${PATH}"
RUN conda install \
    --freeze-installed \
    --channel conda-forge \
    --channel bioconda \
    --file /opt/10x/environment.yaml \
    && conda clean --all \
    && rm /opt/conda/pkgs/ -rf

version: 0.3

phases:
  pre_build:
    commands:
      - echo Logging in to Amazon ECR...
      - aws ecr get-login-password --region "cn-northwest-1"| docker login --username AWS --password-stdin 632427418419.dkr.ecr.cn-northwest-1.amazonaws.com.cn
  build:
    commands:
      - echo Build started on `date`
      - echo Downloading cellranger
      - wget -q https://singlecellanalysis.s3.cn-northwest-1.amazonaws.com.cn/cellranger-6.1.2.tar.gz -O cellranger.tar.gz
      - echo Building the Docker image...
      - docker build -t singlecellanalysis:6.1.0 .
      - docker tag singlecellanalysis:6.1.0 632427418419.dkr.ecr.cn-northwest-1.amazonaws.com.cn/singlecellanalysis:6.1.0
  post_build:
    commands:
      - echo Build completed on `date`
      - echo Pushing the Docker image...
      - docker push 632427418419.dkr.ecr.cn-northwest-1.amazonaws.com.cn/singlecellanalysis:6.1.0

{
  "cellranger_count_GEX.fq2s": [
    "s3://singlecellanalysis/fastq/neuron_1k_v2_S1_L001_R2_001.fastq.gz"
  ],
  "cellranger_count_GEX.sample_name": "singlecellananlysis",
  "cellranger_count_GEX.fq1s": [
    "s3://singlecellanalysis/fastq/neuron_1k_v2_S1_L001_R1_001.fastq.gz"
  ],
  "cellranger_count_GEX.transcriptome": "/usr/local/aws-cli/reference/refdata-gex-mm10-2020-A",
  "cellranger_count_GEX.runtime_attr_count": {
    "cpu": 4,
    "memory_gb": 32,
    "docker": "632427418419.dkr.ecr.cn-northwest-1.amazonaws.com.cn/singlecellanalysis:6.1.0",
    "queue": "arn:aws-cn:batch:cn-northwest-1:632427418419:job-queue/default-singlecellanalysis"
  },
  "cellranger_count_GEX.expect_cells": "3000"
}

version 1.0

struct RuntimeAttr {
  Int? cpu
  Int? memory_gb
  String? docker
  String? queue
}

workflow cellranger_count_GEX {
  input {
    Array[File] fq1s
    Array[File] fq2s
    String sample_name
    Int expect_cells = 3000
    String transcriptome

    RuntimeAttr runtime_attr_count
  }
  call count {
    input:
      fq1s = fq1s,
      fq2s = fq2s,
      sample_name = sample_name,
      expect_cells = expect_cells,
      transcriptome = transcriptome,

      runtime_attr_override = runtime_attr_count
  }
  output {
    File report = count.out_report
    File bam = count.out_bam
    File h5ad = count.out_h5ad
  }
}

task count {
  input {
    Array[File] fq1s
    Array[File] fq2s
    String sample_name
    Int expect_cells
    String transcriptome

    RuntimeAttr runtime_attr_override
  }
  command {
    set -euo pipefail
    python3 /opt/10x/raw_to_tenx.py --fq1s ~{sep=" --fq1s " fq1s} --fq2s ~{sep=" --fq2s " fq2s} --sample_name "~{sample_name}"
    cellranger count --id=~{sample_name} --fastqs=rawdata --sample=~{sample_name} --expect-cells=~{expect_cells} --transcriptome=~{transcriptome} --localmem=~{memory_gb} --nopreflight --disable-ui --nosecondary
    python3 /opt/10x/tenx_to_h5ad.py --h5 "~{sample_name}/outs/filtered_feature_bc_matrix.h5"
  }
  RuntimeAttr runtime_attr_default = object {
    cpu: 1,
    memory_gb: 32
  }
  Int cpu = select_first([runtime_attr_override.cpu, runtime_attr_default.cpu])
  Int memory_gb = select_first([runtime_attr_override.memory_gb, runtime_attr_default.memory_gb])
  runtime {
    cpu: select_first([runtime_attr_override.cpu, runtime_attr_default.cpu])
    memory: select_first([runtime_attr_override.memory_gb, runtime_attr_default.memory_gb])+"GiB"
    docker: select_first([runtime_attr_override.docker, runtime_attr_default.docker])
    queueArn: select_first([runtime_attr_override.queue, runtime_attr_default.queue])
  }
  output {
    File out_report = "~{sample_name}/outs/web_summary.html"
    File out_h5ad = "~{sample_name}/outs/filtered_feature_bc_matrix.h5ad"
    File out_bam = "~{sample_name}/outs/possorted_genome_bam.bam"
  }
}

{
  "outputs": {
    "cellranger_count_GEX.h5ad": "s3://singlecellanalysis/cellranger_count_GEX/a6b403bd-c6fb-4bae-b4cb-ded55d009efd/call-count/singlecellananlysis/outs/filtered_feature_bc_matrix.h5ad",
    "cellranger_count_GEX.report": "s3://singlecellanalysis/cellranger_count_GEX/a6b403bd-c6fb-4bae-b4cb-ded55d009efd/call-count/singlecellananlysis/outs/web_summary.html",
    "cellranger_count_GEX.bam": "s3://singlecellanalysis/cellranger_count_GEX/a6b403bd-c6fb-4bae-b4cb-ded55d009efd/call-count/singlecellananlysis/outs/possorted_genome_bam.bam"
  },
  "id": "a6b403bd-c6fb-4bae-b4cb-ded55d009efd"
}