Merge branch 'dev' for release 3.10.16

.github/workflows/binary.yml

@@ -18,15 +18,28 @@ jobs:
           run: bash -x .github/workflows/get_binary_release.sh
         - name: software version
           run: ./amrfinder --version
-        - name: test protein
+        - name: run tests included with distribution
           run: |
               ./amrfinder --plus -p test_prot.fa -g test_prot.gff -O Escherichia > test_prot.got
               diff test_prot.expected test_prot.got
-        - name: test dna
-          run: |
               ./amrfinder --plus -n test_dna.fa -O Escherichia --mutation_all test_dna_mut_all.got > test_dna.got
               diff test_dna.expected test_dna.got
-        - name: test combined
-          run: |
               ./amrfinder --plus -n test_dna.fa -p test_prot.fa -g test_prot.gff -O Escherichia > test_both.got
               diff test_both.expected test_both.got
+        - name: Run tests
+          run: |
+              # temporarily we need to download the test script. Can remove
+              # once we have a new software release and it's included in the
+              # distribution
+              BASE_URL="https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/master"
+              echo "BASE_URL=${BASE_URL}"
+              curl --silent -L \
+                 -O ${BASE_URL}/test_dna.fa \
+                 -O ${BASE_URL}/test_prot.fa \
+                 -O ${BASE_URL}/test_prot.gff \
+                 -O ${BASE_URL}/test_both.expected \
+                 -O ${BASE_URL}/test_dna.expected \
+                 -O ${BASE_URL}/test_dna_mut_all.expected \
+                 -O ${BASE_URL}/test_prot.expected \
+                 -O ${BASE_URL}/test_amrfinder.sh
+              bash -x ./test_amrfinder.sh

.github/workflows/conda.yml

@@ -7,11 +7,9 @@ on:
     repository_dispatch:
         types: [linux-bioconda-test, install-test]
 jobs:
-  build:
-
+  install_and_test:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
     - name: configure conda
       run: |
           . $CONDA/bin/activate
@@ -30,21 +28,22 @@ jobs:
     - name: download latest AMRFinderPlus database
       run: |
           source /usr/share/miniconda/bin/activate
-          echo CONDA_PREFIX = $CONDA_PREFIX
+          echo CONDA_PREFIX = ${CONDA_PREFIX}
           /usr/share/miniconda/bin/amrfinder --force_update
-    - name: test protein
+    - name: download tests
       run: |
-          source /usr/share/miniconda/bin/activate
-          echo CONDA_PREFIX = $CONDA_PREFIX
-          amrfinder --plus -p test_prot.fa -g test_prot.gff -O Escherichia > test_prot.got
-          diff test_prot.expected test_prot.got
-    - name: test dna
+          BASE_URL=https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/master
+          curl --silent -L \
+             -O ${BASE_URL}/test_dna.fa \
+             -O ${BASE_URL}/test_prot.fa \
+             -O ${BASE_URL}/test_prot.gff \
+             -O ${BASE_URL}/test_both.expected \
+             -O ${BASE_URL}/test_dna.expected \
+             -O ${BASE_URL}/test_dna_mut_all.expected \
+             -O ${BASE_URL}/test_prot.expected \
+             -O ${BASE_URL}/test_amrfinder.sh
+    - name: run tests
       run: |
           source /usr/share/miniconda/bin/activate
-          amrfinder --plus -n test_dna.fa -O Escherichia --mutation_all test_dna_mut_all.got > test_dna.got
-          diff test_dna.expected test_dna.got
-    - name: test combined
-      run: |
-          source /usr/share/miniconda/bin/activate
-          amrfinder --plus -n test_dna.fa -p test_prot.fa -g test_prot.gff -O Escherichia > test_both.got
-          diff test_both.expected test_both.got
+          echo CONDA_PREFIX = $CONDA_PREFIX
+          bash ./test_amrfinder.sh path

.github/workflows/dockerhub.yml

@@ -0,0 +1,39 @@
+name: dockerhub image
+
+on: 
+    schedule:
+        - cron: '15 4 * * *' # 4:15am everyday    
+    workflow_dispatch:
+    repository_dispatch:
+        types: [docker-test, install-test]
+jobs:
+  test_dockerhub:
+    runs-on: ubuntu-latest
+    container: ncbi/amr:latest
+    steps:
+      - run: |
+          echo "GITHUB_REF=${GITHUB_REF}"
+          echo "GITHUB_REPOSTIORY=${GITHUB_REPOSITORY}"
+          echo "GITHUB_SHA=${GITHUB_SHA}"
+          amrfinder --version
+          apt-get install curl
+          # For docker always read off of master branch
+          #BASE_URL="https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/${GITHUB_SHA}/"
+          BASE_URL="https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/master/"
+          echo "BASE_URL=${BASE_URL}"
+          curl --silent -L \
+             -O ${BASE_URL}/test_dna.fa \
+             -O ${BASE_URL}/test_prot.fa \
+             -O ${BASE_URL}/test_prot.gff \
+             -O ${BASE_URL}/test_both.expected \
+             -O ${BASE_URL}/test_dna.expected \
+             -O ${BASE_URL}/test_dna_mut_all.expected \
+             -O ${BASE_URL}/test_prot.expected \
+             -O ${BASE_URL}/test_amrfinder.sh
+          bash ./test_amrfinder.sh path
+        name: Run tests
+        env: 
+          GIT_BRANCH: master
+          GITHUB_REF: ${{env.GITHUB_REF}}
+
+

.github/workflows/mac_conda.yml

@@ -39,18 +39,20 @@ jobs:
       run: |
           source /Users/runner/miniconda3/bin/activate
           /Users/runner/miniconda3/bin/amrfinder -u
-    - name: Test protein
+    - name: Download tests
+      run: |
+          BASE_URL=https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/master
+          curl --silent -L \
+             -O ${BASE_URL}/test_dna.fa \
+             -O ${BASE_URL}/test_prot.fa \
+             -O ${BASE_URL}/test_prot.gff \
+             -O ${BASE_URL}/test_both.expected \
+             -O ${BASE_URL}/test_dna.expected \
+             -O ${BASE_URL}/test_dna_mut_all.expected \
+             -O ${BASE_URL}/test_prot.expected \
+             -O ${BASE_URL}/test_amrfinder.sh
+    - name: Run tests
       run: |
           source /Users/runner/miniconda3/bin/activate
-          amrfinder --plus -p test_prot.fa -g test_prot.gff -O Escherichia > test_prot.got
-          diff test_prot.expected test_prot.got
-    - name: Test dna
-      run: |
-          source /Users/runner/miniconda3/bin/activate
-          amrfinder --plus -n test_dna.fa -O Escherichia --mutation_all test_dna_mut_all.got > test_dna.got
-          diff test_dna.expected test_dna.got
-    - name: Test combined
-      run: |
-          source /Users/runner/miniconda3/bin/activate
-          amrfinder --plus -n test_dna.fa -p test_prot.fa -g test_prot.gff -O Escherichia > test_both.got
-          diff test_both.expected test_both.got
+          echo CONDA_PREFIX = $CONDA_PREFIX
+          bash ./test_amrfinder.sh path

.github/workflows/trigger_action.sh

@@ -11,6 +11,7 @@ Actions that can be triggered:
 3. linux-binary-test
 4. linux-bioconda-test
 5. mac-bioconda-test
+6. docker-test
 END
 
 echo -n "% "
@@ -29,6 +30,7 @@ case $action in
     3|linux-binary-test) event=linux-binary-test;;
     4|linux-bioconda-test) event=linux-bioconda-test;;
     5|mac-bioconda-test) event=mac-bioconda-test;;
+    6|docker-test) event=docker-test;;
     *) echo "Unknown action. Please try again"
         exit 1;;
 esac

Makefile

@@ -175,17 +175,4 @@ src_dist:
 	tar cvfz $(SRC_DIST).tar.gz $(DISTFILES)
 
 test : $(DISTFILES) Makefile *.cpp *.hpp *.inc test_dna.fa test_prot.fa test_prot.gff test_dna.fa test_dna.expected test_prot.expected test_both.expected
-	# curl -O https://raw.githubusercontent.com/ncbi/amr/master/test_dna.fa \
-	# 	-O https://raw.githubusercontent.com/ncbi/amr/master/test_prot.fa \
-	# 	-O https://raw.githubusercontent.com/ncbi/amr/master/test_prot.gff \
-	# 	-O https://raw.githubusercontent.com/ncbi/amr/master/test_both.expected \
-	# 	-O https://raw.githubusercontent.com/ncbi/amr/master/test_dna.expected \
-	# 	-O https://raw.githubusercontent.com/ncbi/amr/master/test_prot.expected
-	./amrfinder --plus -p test_prot.fa -g test_prot.gff -O Escherichia > test_prot.got
-	diff test_prot.expected test_prot.got
-	./amrfinder --plus -n test_dna.fa -O Escherichia --mutation_all test_dna_mut_all.got > test_dna.got
-	diff test_dna.expected test_dna.got
-	# diff test_dna_mut_all.expected test_dna_mut_all.got
-	./amrfinder --plus -n test_dna.fa -p test_prot.fa -g test_prot.gff -O Escherichia > test_both.got
-	diff test_both.expected test_both.got
-
+	./test_amrfinder.sh

alignment.hpp

@@ -146,6 +146,7 @@ struct SeqChange : Root
 	  // !nullptr
 
 	const SeqChange* replacement {nullptr};
+	  // !nullptr => *this is replaced by *replacement
 
 
   SeqChange () = default;
@@ -180,10 +181,8 @@ struct SeqChange : Root
     { return ! len; }
 
 
-private:
   bool hasMutation () const
-    { return ! mutations. empty () /*mutation*/; }
-public:
+    { return ! mutations. empty () && ! empty () && ! replacement; }
   string getMutationStr () const;
   size_t getStop () const
     { return start + len; }
@@ -286,6 +285,12 @@ struct Alignment : Root
     }
 
 
+  bool hasMutation () const
+    { for (const SeqChange& seqChange : seqChanges)
+        if (seqChange. hasMutation ())
+          return true;
+      return false;
+    }
   double pIdentity () const
     { return (double) nident / (double) targetSeq. size (); }
   double refCoverage () const

amr_report.cpp

@@ -682,6 +682,7 @@ struct BlastAlignment : Alignment
   		            td << na;        	
   		        }
   	        }
+  	        IMPLY ((isMutation () && ! seqChange. empty () && mut && ! seqChange. replacement), hasMutation ());
   	        if (   ! isMutation ()
   	            || (! seqChange. empty () && mut && ! seqChange. replacement)  // resistant mutation
   	            || verbose ()
@@ -1051,8 +1052,7 @@ struct BlastAlignment : Alignment
 	      	  && !        insideEq (other)
 	      	 )
 	        return false;
-	    //if (targetProt)
-	      //{ LESS_PART (other, *this, isMutation ()); }
+	      LESS_PART (other, *this, hasMutation ());
 	      LESS_PART (other, *this, refExactlyMatched ());  // PD-1261, PD-1678
 	      LESS_PART (other, *this, nident);
 	      LESS_PART (*this, other, refEffectiveLen ());  
@@ -1103,15 +1103,17 @@ struct BlastAlignment : Alignment
   bool better (const HmmAlignment& other) const
     { ASSERT (other. good ());
     	ASSERT (other. blastAl. get ());
-    	if (! inFam ())
-    	  return false;
+    /*if (! inFam ())
+    	  return false; */
     	if (targetProt)
     	{ if (targetName != other. sseqid)
 	        return false;
 	    }
 	    else
 	    	if (! other. blastAl->matchesCds (*this))
 	    		return false;
+    	if (hasMutation ())
+    	  return true;
       return    refExactlyMatched () 
              || getFam () -> descendantOf (other. fam)
              ;
@@ -1711,8 +1713,11 @@ struct Batch
     	  for (const BlastAlignment* blastAl : it. second)
     	    if (blastAl->better (**hmmIt))
   	      {
-  	        ASSERT (! blastAl->hmmAl);
-  	        var_cast (blastAl) -> hmmAl = *hmmIt;
+  	        if (! blastAl->isMutation ())
+  	        {
+    	        ASSERT (! blastAl->hmmAl);
+    	        var_cast (blastAl) -> hmmAl = *hmmIt;
+    	      }
             hmmIt. erase ();
             found = true;
   	        break;
@@ -2011,7 +2016,7 @@ struct ThisApplication : Application
       addKey ("hmmsearch", "Output of hmmsearch");
       addKey ("organism", "Taxonomy group for mutations");
       addKey ("mutation", "Mutations table");
-      addKey ("susceptible", "Table of susceptibel proteins with resistance cutoffs");
+      addKey ("susceptible", "Table of susceptible proteins with resistance cutoffs");
       addKey ("mutation_all", "File to report all mutations");
       addKey ("suppress_prot", "File with protein accessions to suppress");
       addKey ("ident_min", "Min. identity to the reference protein (0..1). -1 means use a curated threshold if it exists and " + toString (ident_min_def) + " otherwise", "-1");

amrfinder.cpp

@@ -33,6 +33,8 @@
 *               cat, cp, cut, head, ln, mv, sort, tail
 *
 * Release changes:
+*   3.10.16 09/22/2021 PD-3958  point mutations oiverride HMM hits
+*   3.10.15 08/23/2021          -mt_mode 1 is used only for multi-FASTA files: SB-3162
 *   3.10.14 08/19/2021 PD-3918  BLAST output to stderr is not checked
 *   3.10.13 08/19/2021 PD-3918  BLAST -mt_mode 1 is turned off for Mac, see SB-3163
 *   3.10.12 08/19/2021 PD-3918  BLAST output to stderr is reported as error, except for BLASTN due to SB-3162
@@ -220,8 +222,6 @@ constexpr double partial_coverage_min_def = 0.5;
 
 
 
-// ThisApplication
-
 struct ThisApplication : ShellApplication
 {
   ThisApplication ()
@@ -608,7 +608,6 @@ struct ThisApplication : ShellApplication
 
 
 		// PD-3051
-		try
 		{
   	  istringstream versionIss (version);
   		const SoftwareVersion softwareVersion (versionIss);
@@ -621,11 +620,7 @@ struct ThisApplication : ShellApplication
       if (softwareVersion < softwareVersion_min)
         throw runtime_error ("Database requires sofware version at least " + softwareVersion_min. str ());
       if (dataVersion < dataVersion_min)
-        throw runtime_error ("Software requires database version at least " + dataVersion_min. str ());
-    }
-    catch (const exception &e)
-    {
-      throw runtime_error (e. what () + downloadLatestInstr);
+        throw runtime_error ("Software requires database version at least " + dataVersion_min. str () + downloadLatestInstr);
     }
 
 
@@ -888,7 +883,7 @@ struct ThisApplication : ShellApplication
       			const Chronometer_OnePass cop ("blastp", cerr, false, qc_on && ! quiet);
       			// " -task blastp-fast -word_size 6  -threshold 21 "  // PD-2303
       			exec (fullProg ("blastp") + " -query " + prot1 + " -db " + tmp + ".db/AMRProt" + "  " 
-      			      + blastp_par + get_num_threads_param ("blastp", protLen_total / 10000) + " " BLAST_FMT " -out " + tmp + ".blastp > /dev/null 2> " + tmp + ".blastp-err", logFName);
+      			      + blastp_par + get_num_threads_param ("blastp", min (nProt, protLen_total / 10000)) + " " BLAST_FMT " -out " + tmp + ".blastp > /dev/null 2> " + tmp + ".blastp-err", logFName);
       		//checkBlastErr (tmp + ".blastp-err");
       		}
 
@@ -950,7 +945,7 @@ struct ThisApplication : ShellApplication
       			if (blastx == "blastx")
       			{
         			exec (fullProg ("blastx") + "  -query " + dna + " -db " + tmp + ".db/AMRProt" + "  "
-            			  + blastx_par + " " BLAST_FMT " " + get_num_threads_param ("blastx", dnaLen_total / 10002)
+            			  + blastx_par + " " BLAST_FMT " " + get_num_threads_param ("blastx", min (nDna, dnaLen_total / 10002))
             			  + " -out " + tmp + ".blastx > /dev/null 2> " + tmp + ".blastx-err", logFName);
         		//checkBlastErr (tmp + ".blastx-err");
             }
@@ -990,7 +985,7 @@ struct ThisApplication : ShellApplication
       			stderr << "Running blastn...\n";
        			const Chronometer_OnePass cop ("blastn", cerr, false, qc_on && ! quiet);
       			exec (fullProg ("blastn") + " -query " + dna + " -db " + tmp + ".db/AMR_DNA-" + organism1 + " -evalue 1e-20  -dust no  " 
-      			      + get_num_threads_param ("blastn", dnaLen_total / 2500000) + " " BLAST_FMT " -out " + tmp + ".blastn > " + logFName + " 2> " + tmp + ".blastn-err", logFName);
+      			      + get_num_threads_param ("blastn", min (nDna, dnaLen_total / 2500000)) + " " BLAST_FMT " -out " + tmp + ".blastn > " + logFName + " 2> " + tmp + ".blastn-err", logFName);
       		//checkBlastErr (tmp + ".blastn-err");  // SB-3162 ??
       		}
     		}

common.cpp

@@ -1796,9 +1796,9 @@ void Token::readInput (CharInput &in)
 	  {
 	    type = eText;
 	    toNumberDate ();
-	    QC_ASSERT (   type == eInteger
+	  /*QC_ASSERT (   type == eInteger
 	               || type == eDouble
-	              );
+	              );*/
 	  }
 	}
 	else if (isLetter (c))
@@ -2188,7 +2188,7 @@ void TextTable::setHeader ()
   {
     row_num++;
     if (row. size () != header. size ())
-      throw Error (*this, "Row " + to_string (row_num) + " contains " + to_string (row. size ()) + " fields whereas table has " + to_string (header. size ()) + " columns");
+      throw Error (*this, "Row " + to_string (row_num) + " contains " + to_string (row. size ()) + " fields whereas header has " + to_string (header. size ()) + " columns");
     FFOR (RowNum, i, row. size ())
     {
       const string& field = row [i];
@@ -2227,13 +2227,19 @@ void TextTable::qc () const
 {
   if (! qc_on)
     return;
-  Named::qc ();
+  if (! name. empty ())
+    Named::qc (); 
 
   {    
     StringVector v;  v. reserve (header. size ());
-    for (const Header& h : header)
+    FFOR (size_t, i, header. size ())
     {
-      h. qc ();
+      const Header& h = header [i];
+      try { h. qc (); }
+        catch (const exception &e)
+        {
+          throw ("Header column #" + to_string (i + 1) + ": " + e. what ());
+        }
       v << h. name;
     }
     v. sort ();