Track line numbers in copyright files #13

.gitignore

@@ -12,6 +12,7 @@
 /Lib
 /pip-selfcheck.json
 /tmp
+/venv
 .Python
 /include
 /Include

.travis.yml

@@ -13,9 +13,10 @@ python:
   - "3.6"
   - "3.7"
   - "3.8"
+  - "3.9"
 
 # Scripts to run at install stage
 install: ./configure --dev
 
 # Scripts to run at script stage
-script: tmp/bin/pytest
+script: venv/bin/pytest

CHANGELOG.rst

@@ -1,8 +1,23 @@
 Changelog
 =========
 
-v2021.5.12
-----------
+v30.0.0 - 2021-09-07
+------------------------
+
+This is a major incompatible version
+
+- Switch back to semver/semantic versioning
+- Add new deb822 parser for tracking line number in copyright files #13
+- Track line numbers in copyright files to ensure proper line tracking in
+  license detection in https://github.com/nexB/scancode-toolkit/issues/2643
+- Drop sort arguments on debcon objects ``dump/s`` methods. This was not
+  designed consistently and is not used anywhere.
+- Remove deprecated debut module. Use the debian_inspector module instead.
+- Adop latest skeleton: this rename virtual environment directory from ``tmp`` to ``venv``
+
+
+v21.5.25 - 2021-05-25
+------------------------
 
 - Adopt calendar versioning, latest skeleton and various cosmetic code formatting
 - Fix bug where some content could be dropped from invalid debian file

azure-pipelines.yml

@@ -13,52 +13,52 @@ jobs:
           image_name: ubuntu-16.04
           python_versions: ['3.6', '3.7', '3.8', '3.9']
           test_suites:
-              all: tmp/bin/pytest -vvs
+              all: venv/bin/pytest -vvs
 
     - template: etc/ci/azure-posix.yml
       parameters:
           job_name: ubuntu18_cpython
           image_name: ubuntu-18.04
           python_versions: ['3.6', '3.7', '3.8', '3.9']
           test_suites:
-              all: tmp/bin/pytest -n 2 -vvs
+              all: venv/bin/pytest -n 2 -vvs
 
     - template: etc/ci/azure-posix.yml
       parameters:
           job_name: ubuntu20_cpython
           image_name: ubuntu-20.04
           python_versions: ['3.6', '3.7', '3.8', '3.9']
           test_suites:
-              all: tmp/bin/pytest -n 2 -vvs
+              all: venv/bin/pytest -n 2 -vvs
 
     - template: etc/ci/azure-posix.yml
       parameters:
           job_name: macos1014_cpython
           image_name: macos-10.14
           python_versions: ['3.6', '3.7', '3.8', '3.9']
           test_suites:
-              all: tmp/bin/pytest -n 2 -vvs
+              all: venv/bin/pytest -n 2 -vvs
 
     - template: etc/ci/azure-posix.yml
       parameters:
           job_name: macos1015_cpython
           image_name: macos-10.15
           python_versions: ['3.6', '3.7', '3.8', '3.9']
           test_suites:
-              all: tmp/bin/pytest -n 2 -vvs
+              all: venv/bin/pytest -n 2 -vvs
 
     - template: etc/ci/azure-win.yml
       parameters:
           job_name: win2016_cpython
           image_name: vs2017-win2016
           python_versions: ['3.6', '3.7', '3.8', '3.9']
           test_suites:
-              all: tmp\Scripts\pytest -n 2 -vvs
+              all: venv\Scripts\pytest -n 2 -vvs
 
     - template: etc/ci/azure-win.yml
       parameters:
           job_name: win2019_cpython
           image_name: windows-2019
           python_versions: ['3.6', '3.7', '3.8', '3.9']
           test_suites:
-              all: tmp\Scripts\pytest -n 2 -vvs
+              all: venv\Scripts\pytest -n 2 -vvs

configure

@@ -11,7 +11,7 @@ set -e
 #set -x
 
 ################################
-# A configuration script to set things up: 
+# A configuration script to set things up:
 # create a virtualenv and install or update thirdparty packages.
 # Source this script for initial configuration
 # Use configure --help for details
@@ -26,16 +26,16 @@ CLI_ARGS=$1
 ################################
 
 # Requirement arguments passed to pip and used by default or with --dev.
-REQUIREMENTS="--editable ."
-DEV_REQUIREMENTS="--editable .[testing]"
+REQUIREMENTS="--editable . --constraint requirements.txt"
+DEV_REQUIREMENTS="--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt"
 
 # where we create a virtualenv
-VIRTUALENV_DIR=tmp
+VIRTUALENV_DIR=venv
 
 # Cleanable files and directories with the --clean option
 CLEANABLE="
     build
-	tmp"
+	venv"
 
 # extra  arguments passed to pip
 PIP_EXTRA_ARGS=" "
@@ -50,9 +50,12 @@ VIRTUALENV_PYZ_URL=https://bootstrap.pypa.io/virtualenv.pyz
 CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin
 
+# Find packages from the local thirdparty directory or from thirdparty.aboutcode.org
+PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty --find-links https://thirdparty.aboutcode.org/pypi"
+
 
 ################################
-# Set the quiet flag to empty if not defined 
+# Set the quiet flag to empty if not defined
 if [[ "$CFG_QUIET" == "" ]]; then
     CFG_QUIET=" "
 fi
@@ -63,7 +66,7 @@ fi
 # Use environment variables or a file if available.
 # Otherwise the latest Python by default.
 if [[ "$PYTHON_EXECUTABLE" == "" ]]; then
-    # check for a file named PYTHON_EXECUTABLE 
+    # check for a file named PYTHON_EXECUTABLE
     if [ -f "$CFG_ROOT_DIR/PYTHON_EXECUTABLE" ]; then
         PYTHON_EXECUTABLE=$(cat "$CFG_ROOT_DIR/PYTHON_EXECUTABLE")
     else
@@ -78,10 +81,14 @@ cli_help() {
     echo "  usage: ./configure [options]"
     echo
     echo The default is to configure for regular use. Use --dev for development.
+    echo Use the --init option if starting a new project and the project
+    echo dependencies are not available on thirdparty.aboutcode.org/pypi/
+    echo and requirements.txt and/or requirements-dev.txt has not been generated.
     echo
     echo The options are:
     echo " --clean: clean built and installed files and exit."
     echo " --dev:   configure the environment for development."
+    echo " --init:  pull dependencies from PyPI. Used when first setting up a project."
     echo " --help:  display this help message and exit."
     echo
     echo By default, the python interpreter version found in the path is used.
@@ -155,12 +162,22 @@ install_packages() {
 # Main command line entry point
 CFG_DEV_MODE=0
 CFG_REQUIREMENTS=$REQUIREMENTS
-
-case "$CLI_ARGS" in
-    --help)    cli_help;;
-    --clean)   clean;;
-    --dev)     CFG_REQUIREMENTS="$DEV_REQUIREMENTS" && CFG_DEV_MODE=1;;
-esac
+NO_INDEX="--no-index"
+
+# We are using getopts to parse option arguments that start with "-"
+while getopts :-: optchar; do
+    case "${optchar}" in
+        -)
+            case "${OPTARG}" in
+                help  ) cli_help;;
+                clean ) clean;;
+                dev   ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS" && CFG_DEV_MODE=1;;
+                init  ) NO_INDEX="";;
+            esac;;
+    esac
+done
+
+PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS $NO_INDEX"
 
 create_virtualenv "$VIRTUALENV_DIR"
 install_packages "$CFG_REQUIREMENTS"

configure.bat

@@ -9,7 +9,7 @@
 
 
 @rem ################################
-@rem # A configuration script to set things up: 
+@rem # A configuration script to set things up:
 @rem # create a virtualenv and install or update thirdparty packages.
 @rem # Source this script for initial configuration
 @rem # Use configure --help for details
@@ -24,14 +24,14 @@
 @rem ################################
 
 @rem # Requirement arguments passed to pip and used by default or with --dev.
-set "REQUIREMENTS=--editable ."
-set "DEV_REQUIREMENTS=--editable .[testing]"
+set "REQUIREMENTS=--editable . --constraint requirements.txt"
+set "DEV_REQUIREMENTS=--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt"
 
 @rem # where we create a virtualenv
-set "VIRTUALENV_DIR=tmp"
+set "VIRTUALENV_DIR=venv"
 
 @rem # Cleanable files and directories to delete with the --clean option
-set "CLEANABLE=build tmp"
+set "CLEANABLE=build venv"
 
 @rem # extra  arguments passed to pip
 set "PIP_EXTRA_ARGS= "
@@ -48,7 +48,13 @@ set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts"
 
 
 @rem ################################
-@rem # Set the quiet flag to empty if not defined 
+@rem # Thirdparty package locations and index handling
+set "PIP_EXTRA_ARGS=--find-links %CFG_ROOT_DIR%\thirdparty --find-links https://thirdparty.aboutcode.org/pypi" & %INDEX_ARG%
+@rem ################################
+
+
+@rem ################################
+@rem # Set the quiet flag to empty if not defined
 if not defined CFG_QUIET (
     set "CFG_QUIET= "
 )
@@ -58,25 +64,32 @@ if not defined CFG_QUIET (
 @rem # Main command line entry point
 set CFG_DEV_MODE=0
 set "CFG_REQUIREMENTS=%REQUIREMENTS%"
-
-if "%1" EQU "--help"   (goto cli_help)
-if "%1" EQU "--clean"  (goto clean)
-if "%1" EQU "--dev"    (
-    set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%"
-    set CFG_DEV_MODE=1
-)
-if "%1" EQU "--python"  (
-    echo "The --python is now DEPRECATED. Use the PYTHON_EXECUTABLE environment
-    echo "variable instead. Run configure --help for details."
-    exit /b 0
+set "NO_INDEX=--no-index"
+
+:again
+if not "%1" == "" (
+    if "%1" EQU "--help"   (goto cli_help)
+    if "%1" EQU "--clean"  (goto clean)
+    if "%1" EQU "--dev"    (
+        set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%"
+        set CFG_DEV_MODE=1
+    )
+    if "%1" EQU "--init"   (
+        set "NO_INDEX= "
+    )
+    shift
+    goto again
 )
 
+set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% %NO_INDEX%"
+
+
 @rem ################################
 @rem # find a proper Python to run
 @rem # Use environment variables or a file if available.
 @rem # Otherwise the latest Python by default.
 if not defined PYTHON_EXECUTABLE (
-    @rem # check for a file named PYTHON_EXECUTABLE 
+    @rem # check for a file named PYTHON_EXECUTABLE
     if exist ""%CFG_ROOT_DIR%\PYTHON_EXECUTABLE"" (
         set /p PYTHON_EXECUTABLE=<""%CFG_ROOT_DIR%\PYTHON_EXECUTABLE""
     ) else (
@@ -142,6 +155,9 @@ if %ERRORLEVEL% neq 0 (
     %PIP_EXTRA_ARGS% ^
     %CFG_REQUIREMENTS%
 
+@rem # Create junction to bin to have the same directory between linux and windows
+mklink /J %CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin %CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts
+
 if %ERRORLEVEL% neq 0 (
     exit /b %ERRORLEVEL%
 )
@@ -156,10 +172,14 @@ exit /b 0
     echo "  usage: configure [options]"
     echo " "
     echo The default is to configure for regular use. Use --dev for development.
+    echo Use the --init option if starting a new project and the project
+    echo dependencies are not available on thirdparty.aboutcode.org/pypi/
+    echo and requirements.txt and/or requirements-dev.txt has not been generated.
     echo " "
     echo The options are:
     echo " --clean: clean built and installed files and exit."
     echo " --dev:   configure the environment for development."
+    echo " --init:  pull dependencies from PyPI. Used when first setting up a project."
     echo " --help:  display this help message and exit."
     echo " "
     echo By default, the python interpreter version found in the path is used.