Autofix b without p and other minor issues

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.0.0-beta.16
+current_version = 3.0.0-beta.17
 commit = True
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)\-(?P<release>\w+).(?P<num>\d+)

docs/Dev_notes.md

@@ -30,15 +30,37 @@ bumpversion --new-version 3.0.0-alpha.28 num
 The github action is configured to automatically build and publish to PyPI and NPM upon a github code release. Make sure to use same version number in the python module bump command, and github release.
 
 ## Run tests
-To check Syntax trees in Grammar module
+### To check Syntax trees in Grammar module
 ```bash
 cd tree-sitter-usfm3
 export PATH=$PATH:./node_modules/.bin
 tree-sitter generate
 tree-sitter test
 ```
 
-In python module alone
+Ensure the other modules have this newer grammar.
+
+Python:
+```bash
+cd py-usfm-parser
+pip install ../tree-sitter-usfm3/
+```
+
+Node:
+```bash
+cd node-usfm-parser
+npm install ../tree-sitter-usfm3/ --no-save
+```
+
+Web:
+```bash
+cd tree-sitter-usfm3
+tree-sitter build --wasm
+cp tree-sitter-usfm3.wasm ../web-usfm-parser/tree-sitter-usfm.wasm
+```
+
+
+### In python module alone
 
 ```bash
 cd py-usfm-parser
@@ -49,7 +71,7 @@ pytest -k "not compare_usx_with_testsuite_samples and not testsuite_usx_with_rnc
 
 ```
 
-In node module:
+### In node module:
 
 ```bash
 cd node-usfm-parser
@@ -60,7 +82,7 @@ node_modules/mocha/bin/mocha.js --timeout 40000 --grep "Compare" --bail
 node_modules/mocha/bin/mocha.js --timeout 40000 test/basic.js
 ```
 
-In web module:
+### In web module:
 
 ```bash
 cd web-usfm-parser

node-usfm-parser/package.json

@@ -1,6 +1,6 @@
 {
   "name": "usfm-grammar",
-  "version": "3.0.0-beta.16",
+  "version": "3.0.0-beta.17",
   "description": "Uses the tree-sitter-usfm3 parser to convert USFM files to other formats such as USJ, USX, and CSV, and converts them back to USFM",
   "main": "./dist/cjs/index.cjs",
   "module": "./dist/es/index.mjs",
@@ -28,7 +28,7 @@
   "dependencies": {
     "ajv": "^8.17.1",
     "tree-sitter": "0.21.1",
-    "tree-sitter-usfm3": "3.0.0-beta.16",
+    "tree-sitter-usfm3": "3.0.0-beta.17",
     "xmldom": "^0.6.0",
     "xpath": "^0.0.34"
   },

node-usfm-parser/src/validator.js

@@ -167,6 +167,15 @@ class Validator {
                 modifiedUSFM = modifiedUSFM.replace(toReplace, `${toReplace}\\p\n`);
                 changed = true;
             }
+            else if (error.isError && !errorText.startsWith("\\") && error.previousSibling.type === "chapter" &&
+                !error.children.some(ch => ch.type === "paragraph")) {
+                // console.log("Match 7.1");
+                const start = error.previousSibling.startIndex;
+                const end = error.startIndex;
+                const toReplace = modifiedUSFM.slice(start, end);
+                modifiedUSFM = modifiedUSFM.replace(toReplace, `${toReplace}\\p\n`);
+                changed = true;
+            }
             // Stray slash not with a valid marker
             else if (errorText.startsWith("\\") && !validMarkersPattern.test(errorText)) {
                 // console.log("Match 8");

py-usfm-parser/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "usfm-grammar"
-version = "3.0.0-beta.16"
+version = "3.0.0-beta.17"
 description = "Python parser for USFM files, based on tree-sitter-usfm3"
 readme = "README.md"
 authors = [{ name = "BCS Team", email = "[email protected]" }]
@@ -23,7 +23,7 @@ classifiers = [
 keywords = ["usfm", "parser", "grammar", "tree-sitter"]
 dependencies = [
     'tree-sitter==0.22.3; python_version >= "3.9"',
-    'tree-sitter-usfm3==3.0.0-beta.16; python_version >="3.8"',
+    'tree-sitter-usfm3==3.0.0-beta.17; python_version >="3.8"',
     'lxml==5.2.2; python_version >= "3.5"',
     'jsonschema==4.23.0; python_version>= "3.8"'
 ]

py-usfm-parser/requirements.txt

@@ -1,4 +1,4 @@
 tree-sitter==0.22.3
-tree-sitter-usfm3==3.0.0-beta.16
+tree-sitter-usfm3==3.0.0-beta.17
 lxml==5.2.2
 jsonschema==4.23.0

py-usfm-parser/setup.py

@@ -7,7 +7,7 @@ def has_ext_modules(self):
 
 setup(
     name="usfm-grammar",  # Required
-    version="3.0.0-beta.16",  # Required
+    version="3.0.0-beta.17",  # Required
     python_requires=">=3.10",
     # install_requires=["tree-sitter==0.22.3",
                         # "tree-sitter-usfm3==3.0.0-beta.7",  

py-usfm-parser/src/usfm_grammar/__init__.py

@@ -9,4 +9,4 @@
 
 Validator = validator.Validator
 
-__version__ = "3.0.0-beta.16"
+__version__ = "3.0.0-beta.17"

py-usfm-parser/src/usfm_grammar/usfm_parser.py

@@ -233,7 +233,7 @@ def to_usx(self, ignore_errors=False):
 
 
         usx_root = etree.Element("usx")
-        usx_root.set("version", "3.0")
+        usx_root.set("version", "3.1")
         try:
             usx_generator = USXGenerator(USFM_LANGUAGE, self.usfm_bytes, usx_root)
             usx_generator.node_2_usx(self.syntax_tree, usx_root)

py-usfm-parser/src/usfm_grammar/validator.py

@@ -158,6 +158,18 @@ def auto_fix_usfm(self, usfm, fixed=False): #pylint: disable=R0912,R0915
                 # self.modified_usfm = re.sub(to_replace, repalcement, self.modified_usfm)
                 self.modified_usfm = self.modified_usfm.replace(to_replace, repalcement)
                 changed = True
+            elif error.is_error and \
+               (not error_text.startswith("\\")) and \
+               error.prev_sibling.type == "chapter" and \
+               "paragraph" not in [ch.type for ch in error.children]:
+                # print("match 7.1")
+                start = error.prev_sibling.start_byte
+                end = error.start_byte
+                to_replace = self.usfm_bytes[start:end].decode('utf-8')
+                repalcement = to_replace+"\\p\n"
+                # self.modified_usfm = re.sub(to_replace, repalcement, self.modified_usfm)
+                self.modified_usfm = self.modified_usfm.replace(to_replace, repalcement)
+                changed = True
             # Stray slash not with a valid marker
             elif error_text.startswith("\\") and \
                  (not re.match(valid_markers_pattern, error_text)):

tests/autofix/b_without_p.usfm

@@ -0,0 +1,24 @@
+\id GEN genesis Some desc
+\c 1
+\p
+\v 1 test verse
+\s some poem
+\q1
+first line
+\b
+\q1
+\v 2 more lines
+\b
+remaining verses
+\v 3 text follows
+\c 2
+\p
+\v 1 test verse
+\s some poem
+\q1
+first line
+\b
+\q1
+\v 2 more lines
+\b
+\v 3 verse follows

tree-sitter-usfm3/grammar.js

@@ -228,7 +228,11 @@ module.exports = grammar({
     r: $ => seq("\\r ", $.text), // ocurs under c too
 
     sp: $ => seq("\\sp ", $.text),
-    d: $ => seq("\\d ", $.text),
+    d: $ => prec.right(0, seq("\\d ", repeat1(choice($.text,
+      $.footnote, $.crossref,
+      $._characterMarker,
+      $.fig,
+      )))),
     sdBlock: $ => prec.right(0, repeat1($.sd)),
     sd: $ => seq($.sdTag),
     sdTag: $ => seq("\\sd", optional($.numberedLevelMax4), $._spaceOrLine),

tree-sitter-usfm3/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tree-sitter-usfm3",
-  "version": "3.0.0-beta.16",
+  "version": "3.0.0-beta.17",
   "description": "Grammar representation and parser for USFM language using tree-sitter",
   "main": "bindings/node",
   "types": "bindings/node",

tree-sitter-usfm3/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "tree-sitter-usfm3"
 description = "Usfm3 grammar for tree-sitter"
-version = "3.0.0-beta.16"
+version = "3.0.0-beta.17"
 keywords = ["incremental", "parsing", "tree-sitter", "usfm3"]
 classifiers = [
   "Intended Audience :: Developers",

web-usfm-parser/README.md

@@ -23,10 +23,10 @@ function App() {
 
   useEffect(() => {
     const initParser = async () => {
-      await USFMParser.init("https://cdn.jsdelivr.net/npm/[email protected].16/tree-sitter-usfm.wasm",
-                            "https://cdn.jsdelivr.net/npm/[email protected].16/tree-sitter.wasm");
-      await Validator.init("https://cdn.jsdelivr.net/npm/[email protected].16/tree-sitter-usfm.wasm",
-                            "https://cdn.jsdelivr.net/npm/[email protected].16/tree-sitter.wasm");
+      await USFMParser.init("https://cdn.jsdelivr.net/npm/[email protected].17/tree-sitter-usfm.wasm",
+                            "https://cdn.jsdelivr.net/npm/[email protected].17/tree-sitter.wasm");
+      await Validator.init("https://cdn.jsdelivr.net/npm/[email protected].17/tree-sitter-usfm.wasm",
+                            "https://cdn.jsdelivr.net/npm/[email protected].17/tree-sitter.wasm");
 
     };
     initParser();
@@ -60,13 +60,13 @@ It can be used directly in the HTML script tag too. Please ensure its dependenci
 
 ```html
 <script type="module">
-  import { USFMParser, Filter, Validator } from 'https://cdn.jsdelivr.net/npm/[email protected].16/dist/bundle.mjs';
+  import { USFMParser, Filter, Validator } from 'https://cdn.jsdelivr.net/npm/[email protected].17/dist/bundle.mjs';
   console.log('Hello world');
   (async () => {
-  await USFMParser.init("https://cdn.jsdelivr.net/npm/[email protected].16/tree-sitter-usfm.wasm",
-                            "https://cdn.jsdelivr.net/npm/[email protected].16/tree-sitter.wasm");
-  await Validator.init("https://cdn.jsdelivr.net/npm/[email protected].16/tree-sitter-usfm.wasm",
-                            "https://cdn.jsdelivr.net/npm/[email protected].16/tree-sitter.wasm");
+  await USFMParser.init("https://cdn.jsdelivr.net/npm/[email protected].17/tree-sitter-usfm.wasm",
+                            "https://cdn.jsdelivr.net/npm/[email protected].17/tree-sitter.wasm");
+  await Validator.init("https://cdn.jsdelivr.net/npm/[email protected].17/tree-sitter-usfm.wasm",
+                            "https://cdn.jsdelivr.net/npm/[email protected].17/tree-sitter.wasm");
   const usfmParser = new USFMParser('\\id GEN\n\\c 1\n\\p\n\\v 1 In the begining..\\v 2 more text')
   const output = usfmParser.toUSJ()
   console.log({ output })

web-usfm-parser/package.json

@@ -1,6 +1,6 @@
 {
   "name": "usfm-grammar-web",
-  "version": "3.0.0-beta.16",
+  "version": "3.0.0-beta.17",
   "description": "Uses the tree-sitter-usfm3 parser to convert USFM files to other formats such as USJ, USX, and CS, and converts them back to USFM.",
   "type": "module",
   "module": "dist/bundle.mjs",

web-usfm-parser/src/validator.js

@@ -178,6 +178,15 @@ class Validator {
                 modifiedUSFM = modifiedUSFM.replace(toReplace, `${toReplace}\\p\n`);
                 changed = true;
             }
+            else if (error.isError && !errorText.startsWith("\\") && error.previousSibling.type === "chapter" &&
+                !error.children.some(ch => ch.type === "paragraph")) {
+                // console.log("Match 7.1");
+                const start = error.previousSibling.startIndex;
+                const end = error.startIndex;
+                const toReplace = modifiedUSFM.slice(start, end);
+                modifiedUSFM = modifiedUSFM.replace(toReplace, `${toReplace}\\p\n`);
+                changed = true;
+            }
             // Stray slash not with a valid marker
             else if (errorText.startsWith("\\") && !validMarkersPattern.test(errorText)) {
                 // console.log("Match 8");