Upgrade dependencies (especially fastp) to address deadlock problem of fastp

Author: philiptzou

Dockerfile

@@ -1,60 +1,60 @@
-FROM python:3.9-alpine as builder
+FROM python:3.11-alpine as builder
 RUN apk add gcc make g++ zlib-dev bzip2-dev xz-dev linux-headers ncurses-dev curl-dev coreutils
 
 FROM builder as samtools_builder
-ARG SAMTOOLS_VERSION=1.10
+ARG SAMTOOLS_VERSION=1.17
 RUN wget -O /tmp/samtools.tar.bz2 "https://github.com/samtools/samtools/releases/download/${SAMTOOLS_VERSION}/samtools-${SAMTOOLS_VERSION}.tar.bz2" && \
     tar -xjC /tmp -f /tmp/samtools.tar.bz2 && \
     cd /tmp/samtools-${SAMTOOLS_VERSION} && \
     ./configure --prefix=/usr/local/samtools && \
     make && make install
 
 FROM builder as minimap2_installer
-ARG MINIMAP2_VERSION=2.22
+ARG MINIMAP2_VERSION=2.26
 RUN wget -O /tmp/minimap2.tar.bz2 "https://github.com/lh3/minimap2/releases/download/v${MINIMAP2_VERSION}/minimap2-${MINIMAP2_VERSION}_x64-linux.tar.bz2" && \
     tar -xjC /tmp -f /tmp/minimap2.tar.bz2 && \
     mv /tmp/minimap2-${MINIMAP2_VERSION}_x64-linux /usr/local/minimap2
 
 FROM builder as pysam_builder
-RUN echo 'manylinux2014_compatible = True' > /usr/local/lib/python3.9/site-packages/_manylinux.py
-RUN pip install 'cython==0.29.32'
-RUN pip install 'pysam==0.20.0'
+RUN echo 'manylinux2014_compatible = True' > /usr/local/lib/python3.11/site-packages/_manylinux.py
+RUN pip install 'cython==0.29.35'
+RUN pip install 'pysam==0.21.0'
 
 FROM builder as orjson_builder
-RUN echo 'manylinux2014_compatible = True' > /usr/local/lib/python3.9/site-packages/_manylinux.py
+RUN echo 'manylinux2014_compatible = True' > /usr/local/lib/python3.11/site-packages/_manylinux.py
 RUN apk add rust cargo patchelf
-RUN pip install 'orjson==3.8.0'
+RUN pip install 'orjson==3.9.1'
 
 FROM builder as cutadapt_builder
-RUN echo 'manylinux2014_compatible = True' > /usr/local/lib/python3.9/site-packages/_manylinux.py
-RUN pip install 'cutadapt==4.2'
+RUN echo 'manylinux2014_compatible = True' > /usr/local/lib/python3.11/site-packages/_manylinux.py
+RUN pip install 'cutadapt==4.4'
 
 FROM builder as pydep_builder
 COPY --from=pysam_builder /root/.cache/ /root/.cache/
 COPY --from=orjson_builder /root/.cache/ /root/.cache/
 COPY --from=cutadapt_builder /root/.cache/ /root/.cache/
-RUN echo 'manylinux2014_compatible = True' > /usr/local/lib/python3.9/site-packages/_manylinux.py
-RUN pip install 'cython==0.29.32'
+RUN echo 'manylinux2014_compatible = True' > /usr/local/lib/python3.11/site-packages/_manylinux.py
+RUN pip install 'cython==0.29.35'
 COPY requirements.txt /codfreq/
 RUN pip install -r /codfreq/requirements.txt
 
 
 FROM builder as py_builder
 COPY --from=pydep_builder /root/.cache/ /root/.cache/
-RUN echo 'manylinux2014_compatible = True' > /usr/local/lib/python3.9/site-packages/_manylinux.py
-RUN pip install 'cython==0.29.32'
+RUN echo 'manylinux2014_compatible = True' > /usr/local/lib/python3.11/site-packages/_manylinux.py
+RUN pip install 'cython==0.29.35'
 COPY . /codfreq/
 RUN pip install --ignore-installed --target /python-packages /codfreq
 RUN mv /python-packages/bin /python-scripts
 
 FROM builder as fastp_installer
-ARG FASTP_VERSION=0.23.2
+ARG FASTP_VERSION=0.23.4
 RUN wget -O /usr/local/bin/fastp http://opengene.org/fastp/fastp.${FASTP_VERSION}
 RUN chmod +x /usr/local/bin/fastp
 
 FROM builder as htslib_builder
 RUN apk add automake autoconf
-ARG HTSLIB_VERSION=1.16
+ARG HTSLIB_VERSION=1.17
 RUN wget -O /tmp/htslib.tar.gz https://github.com/samtools/htslib/releases/download/${HTSLIB_VERSION}/htslib-${HTSLIB_VERSION}.tar.bz2
 RUN tar -xjC /tmp -f /tmp/htslib.tar.gz
 RUN cd /tmp/htslib-${HTSLIB_VERSION} && \
@@ -65,7 +65,7 @@ RUN cd /tmp/htslib-${HTSLIB_VERSION} && \
 FROM builder as ivar_builder
 RUN apk add automake autoconf
 COPY --from=htslib_builder /usr/local/htslib/ /usr/local/htslib/
-ARG IVAR_VERSION=1.3.1
+ARG IVAR_VERSION=1.4.2
 RUN wget -O /tmp/ivar.tar.gz https://github.com/andersen-lab/ivar/archive/refs/tags/v${IVAR_VERSION}.tar.gz
 RUN tar -xzC /tmp -f /tmp/ivar.tar.gz
 RUN export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/htslib/lib && \
@@ -77,12 +77,12 @@ RUN export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/htslib/lib && \
     ./configure --prefix=/usr/local/ivar && \
     make && make install
 
-FROM python:3.9-alpine
+FROM python:3.11-alpine
 ENV LANG="C.UTF-8" \
     HTSLIB_CONFIGURE_OPTIONS="--enable-plugins"
 RUN apk add --no-cache bash libc6-compat libcurl jq zip pigz
 COPY --from=py_builder /python-scripts/ /usr/local/bin/
-COPY --from=py_builder /python-packages/ /usr/local/lib/python3.9/site-packages/
+COPY --from=py_builder /python-packages/ /usr/local/lib/python3.11/site-packages/
 COPY --from=orjson_builder /usr/lib/libgcc_s.so.1 /usr/lib/libgcc_s.so.1
 COPY --from=samtools_builder /usr/local/samtools/ /usr/local/samtools/
 COPY --from=minimap2_installer /usr/local/minimap2/ /usr/local/minimap2/

Makefile

@@ -1,6 +1,5 @@
 requirements.txt: Pipfile Pipfile.lock
-	@pipenv lock --requirements > requirements.txt
-	# @sed -i '' -E 's/; (sys_platform|python_(full_)?version).*$$//' requirements.txt
+	@pipenv requirements > requirements.txt
 
 login-ecr:
 	@aws ecr get-login-password --region us-west-2 | docker login \

Pipfile

@@ -8,18 +8,18 @@ verify_ssl = true
 boto3 = "*"
 
 [packages]
-cython = "*"
-pysam = ">=0.19.1"
+cython = ">=0.29.35"
+pysam = ">=0.21.0"
 click = "*"
 tqdm = "*"
 more-itertools = "*"
 awscli = "*"
 urllib3 = ">=1.26.5"
 Pygments = ">=2.7.4"
 PyYAML = ">=5.4"
-post-align = {file = "https://github.com/hivdb/post-align/archive/59e7285942a8b42b4b0f1b91a3902f1ac8c7bea4.zip"}
+post-align = {file = "https://github.com/hivdb/post-align/archive/cb28b83e2d9639960533f805f7dc2612ea63ddc6.zip"}
 typing-extensions = "*"
-cutadapt = ">=4.1"
+cutadapt = ">=4.4"
 
 [requires]
-python_version = "3.9"
+python_version = "3.11"