Add summaries

.github/workflows/python-package.yml

@@ -3,11 +3,7 @@
 
 name: Python package
 
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    branches: [ master ]
+on: push
 
 jobs:
   build:

pairtools/__init__.py

@@ -12,4 +12,4 @@
 
 __version__ = "1.0.0-dev1"
 
-# from . import lib
+# from . import lib

pairtools/cli/__init__.py

@@ -142,6 +142,7 @@ def _excepthook(exc_type, value, tb):
 
         sys.excepthook = _excepthook
 
+
 def common_io_options(func):
     @click.option(
         "--nproc-in",
@@ -200,5 +201,5 @@ def wrapper(*args, **kwargs):
     sample,
     filterbycov,
     header,
-    scaling
+    scaling,
 )

pairtools/cli/dedup.py

@@ -5,8 +5,8 @@
 import ast
 import pathlib
 
-# from distutils.log import warn
-# import warnings
+from .._logging import get_logger
+logger = get_logger()
 
 from ..lib import fileio, pairsam_format, headerops
 from . import cli, common_io_options
@@ -60,7 +60,16 @@
     " If the path ends with .gz or .lz4, the output is bgzip-/lz4c-compressed."
     " By default, statistics are not printed.",
 )
-
+@click.option(
+    "--output-bytile-stats",
+    type=str,
+    default="",
+    help="output file for duplicate statistics."
+    " If file exists, it will be open in the append mode."
+    " If the path ends with .gz or .lz4, the output is bgzip-/lz4c-compressed."
+    " By default, by-tile duplicate statistics are not printed."
+    " Note that the readID should be provided and contain tile information for this option. ",
+)
 ### Set the dedup method:
 @click.option(
     "--max-mismatch",
@@ -223,6 +232,7 @@ def dedup(
     output_dups,
     output_unmapped,
     output_stats,
+    output_bytile_stats,
     chunksize,
     carryover,
     max_mismatch,
@@ -260,6 +270,7 @@ def dedup(
         output_dups,
         output_unmapped,
         output_stats,
+        output_bytile_stats,
         chunksize,
         carryover,
         max_mismatch,
@@ -293,6 +304,7 @@ def dedup_py(
     output_dups,
     output_unmapped,
     output_stats,
+    output_bytile_stats,
     chunksize,
     carryover,
     max_mismatch,
@@ -350,8 +362,21 @@ def dedup_py(
         else None
     )
 
+    bytile_dups = False
+    if output_bytile_stats:
+        out_bytile_stats_stream = fileio.auto_open(
+                output_bytile_stats,
+                mode="w",
+                nproc=kwargs.get("nproc_out"),
+                command=kwargs.get("cmd_out", None),
+        )
+        bytile_dups = True
+        if not keep_parent_id:
+            logger.warning("Force output --parent-readID because --output-bytile-stats provided.")
+            keep_parent_id = True
+
     # generate empty PairCounter if stats output is requested:
-    out_stat = PairCounter() if output_stats else None
+    out_stat = PairCounter(bytile_dups=bytile_dups) if output_stats else None
 
     if not output_dups:
         outstream_dups = None
@@ -465,6 +490,9 @@ def dedup_py(
     if out_stat:
         out_stat.save(out_stats_stream)
 
+    if bytile_dups:
+        out_stat.save_bytile_dups(out_bytile_stats_stream)
+
     if instream != sys.stdin:
         instream.close()
 

pairtools/cli/stats.py

@@ -10,6 +10,8 @@
 
 from ..lib.stats import PairCounter, do_merge
 
+from .._logging import get_logger
+logger = get_logger()
 
 UTIL_NAME = "pairtools_stats"
 
@@ -40,8 +42,26 @@
     default=False,
     help="Output stats in yaml format instead of table. ",
 )
+@click.option(
+    "--bytile-dups/--no-bytile-dups",
+    default=False,
+    help="If enabled, will analyse by-tile duplication statistics to estimate"
+    " library complexity more accurately."
+    " Requires parent_readID column to be saved by dedup (will be ignored otherwise)"
+    " Saves by-tile stats into --output_bytile-stats stream, or regular output if --output_bytile-stats is not provided.",
+)
+@click.option(
+    "--output-bytile-stats",
+    default="",
+    required=False,
+    help="output file for tile duplicate statistics."
+    " If file exists, it will be open in the append mode."
+    " If the path ends with .gz or .lz4, the output is bgzip-/lz4c-compressed."
+    " By default, by-tile duplicate statistics are not printed."
+    " Note that the readID and parent_readID should be provided and contain tile information for this option.",
+)
 @common_io_options
-def stats(input_path, output, merge, **kwargs):
+def stats(input_path, output, merge, bytile_dups, output_bytile_stats, **kwargs):
     """Calculate pairs statistics.
 
     INPUT_PATH : by default, a .pairs/.pairsam file to calculate statistics.
@@ -51,10 +71,15 @@ def stats(input_path, output, merge, **kwargs):
 
     The files with paths ending with .gz/.lz4 are decompressed by bgzip/lz4c.
     """
-    stats_py(input_path, output, merge, **kwargs)
+
+    stats_py(
+        input_path, output, merge, bytile_dups, output_bytile_stats, **kwargs,
+    )
 
 
-def stats_py(input_path, output, merge, **kwargs):
+def stats_py(
+    input_path, output, merge, bytile_dups, output_bytile_stats, **kwargs
+):
     if merge:
         do_merge(output, input_path, **kwargs)
         return
@@ -74,21 +99,35 @@ def stats_py(input_path, output, merge, **kwargs):
         nproc=kwargs.get("nproc_out"),
         command=kwargs.get("cmd_out", None),
     )
+    if bytile_dups and not output_bytile_stats:
+        output_bytile_stats = outstream
+    if output_bytile_stats:
+        bytile_dups = True
 
     header, body_stream = headerops.get_header(instream)
     cols = headerops.extract_column_names(header)
 
+    # Check necessary columns for reporting by-tile stats:
+    if bytile_dups and "parent_readID" not in cols:
+        logger.warning(
+            "No 'parent_readID' column in the file, not generating duplicate stats."
+        )
+        bytile_dups = False
+
     # new stats class stuff would come here ...
-    stats = PairCounter()
+    stats = PairCounter(bytile_dups=bytile_dups)
 
-    # collecting statistics
+    # Collecting statistics
     for chunk in pd.read_table(body_stream, names=cols, chunksize=100_000):
         stats.add_pairs_from_dataframe(chunk)
 
     if kwargs.get("with_chromsizes", True):
         chromsizes = headerops.extract_chromsizes(header)
         stats.add_chromsizes(chromsizes)
 
+    if bytile_dups:
+        stats.save_bytile_dups(output_bytile_stats)
+
     # save statistics to file ...
     stats.save(outstream, yaml=kwargs.get("yaml", False))
 

pairtools/lib/dedup.py

@@ -1,6 +1,5 @@
 import numpy as np
 import pandas as pd
-import warnings
 
 import scipy.spatial
 from scipy.sparse import coo_matrix
@@ -380,7 +379,7 @@ def streaming_dedup_cython(
 
         # take care of empty lines not at the end of the file separately
         if rawline and (not stripline):
-            warnings.warn("Empty line detected not at the end of the file")
+            logger.warning("Empty line detected not at the end of the file")
             continue
 
         if stripline: