diff --git a/.bumpversion.cfg b/.bumpversion.cfg index c9d906c6..8de96ef8 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.0.0-beta.16 +current_version = 3.0.0-beta.17 commit = True tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)\-(?P\w+).(?P\d+) diff --git a/docs/Dev_notes.md b/docs/Dev_notes.md index 91e60999..fccc066f 100644 --- a/docs/Dev_notes.md +++ b/docs/Dev_notes.md @@ -30,7 +30,7 @@ bumpversion --new-version 3.0.0-alpha.28 num The github action is configured to automatically build and publish to PyPI and NPM upon a github code release. Make sure to use same version number in the python module bump command, and github release. ## Run tests -To check Syntax trees in Grammar module +### To check Syntax trees in Grammar module ```bash cd tree-sitter-usfm3 export PATH=$PATH:./node_modules/.bin @@ -38,7 +38,29 @@ tree-sitter generate tree-sitter test ``` -In python module alone +Ensure the other modules have this newer grammar. + +Python: +```bash +cd py-usfm-parser +pip install ../tree-sitter-usfm3/ +``` + +Node: +```bash +cd node-usfm-parser +npm install ../tree-sitter-usfm3/ --no-save +``` + +Web: +```bash +cd tree-sitter-usfm3 +tree-sitter build --wasm +cp tree-sitter-usfm3.wasm ../web-usfm-parser/tree-sitter-usfm.wasm +``` + + +### In python module alone ```bash cd py-usfm-parser @@ -49,7 +71,7 @@ pytest -k "not compare_usx_with_testsuite_samples and not testsuite_usx_with_rnc ``` -In node module: +### In node module: ```bash cd node-usfm-parser @@ -60,7 +82,7 @@ node_modules/mocha/bin/mocha.js --timeout 40000 --grep "Compare" --bail node_modules/mocha/bin/mocha.js --timeout 40000 test/basic.js ``` -In web module: +### In web module: ```bash cd web-usfm-parser diff --git a/node-usfm-parser/package.json b/node-usfm-parser/package.json index 9b629bf8..072c9735 100644 --- a/node-usfm-parser/package.json +++ b/node-usfm-parser/package.json @@ -1,6 +1,6 @@ { "name": "usfm-grammar", - "version": "3.0.0-beta.16", + "version": "3.0.0-beta.17", "description": "Uses the tree-sitter-usfm3 parser to convert USFM files to other formats such as USJ, USX, and CSV, and converts them back to USFM", "main": "./dist/cjs/index.cjs", "module": "./dist/es/index.mjs", @@ -28,7 +28,7 @@ "dependencies": { "ajv": "^8.17.1", "tree-sitter": "0.21.1", - "tree-sitter-usfm3": "3.0.0-beta.16", + "tree-sitter-usfm3": "3.0.0-beta.17", "xmldom": "^0.6.0", "xpath": "^0.0.34" }, diff --git a/node-usfm-parser/src/validator.js b/node-usfm-parser/src/validator.js index a4281215..beebdb3e 100644 --- a/node-usfm-parser/src/validator.js +++ b/node-usfm-parser/src/validator.js @@ -167,6 +167,15 @@ class Validator { modifiedUSFM = modifiedUSFM.replace(toReplace, `${toReplace}\\p\n`); changed = true; } + else if (error.isError && !errorText.startsWith("\\") && error.previousSibling.type === "chapter" && + !error.children.some(ch => ch.type === "paragraph")) { + // console.log("Match 7.1"); + const start = error.previousSibling.startIndex; + const end = error.startIndex; + const toReplace = modifiedUSFM.slice(start, end); + modifiedUSFM = modifiedUSFM.replace(toReplace, `${toReplace}\\p\n`); + changed = true; + } // Stray slash not with a valid marker else if (errorText.startsWith("\\") && !validMarkersPattern.test(errorText)) { // console.log("Match 8"); diff --git a/py-usfm-parser/pyproject.toml b/py-usfm-parser/pyproject.toml index 6f7a8152..3982ab3f 100644 --- a/py-usfm-parser/pyproject.toml +++ b/py-usfm-parser/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "usfm-grammar" -version = "3.0.0-beta.16" +version = "3.0.0-beta.17" description = "Python parser for USFM files, based on tree-sitter-usfm3" readme = "README.md" authors = [{ name = "BCS Team", email = "joel@bridgeconn.com" }] @@ -23,7 +23,7 @@ classifiers = [ keywords = ["usfm", "parser", "grammar", "tree-sitter"] dependencies = [ 'tree-sitter==0.22.3; python_version >= "3.9"', - 'tree-sitter-usfm3==3.0.0-beta.16; python_version >="3.8"', + 'tree-sitter-usfm3==3.0.0-beta.17; python_version >="3.8"', 'lxml==5.2.2; python_version >= "3.5"', 'jsonschema==4.23.0; python_version>= "3.8"' ] diff --git a/py-usfm-parser/requirements.txt b/py-usfm-parser/requirements.txt index 6a33450c..6046ad7b 100644 --- a/py-usfm-parser/requirements.txt +++ b/py-usfm-parser/requirements.txt @@ -1,4 +1,4 @@ tree-sitter==0.22.3 -tree-sitter-usfm3==3.0.0-beta.16 +tree-sitter-usfm3==3.0.0-beta.17 lxml==5.2.2 jsonschema==4.23.0 diff --git a/py-usfm-parser/setup.py b/py-usfm-parser/setup.py index c4e31118..0d39589a 100644 --- a/py-usfm-parser/setup.py +++ b/py-usfm-parser/setup.py @@ -7,7 +7,7 @@ def has_ext_modules(self): setup( name="usfm-grammar", # Required - version="3.0.0-beta.16", # Required + version="3.0.0-beta.17", # Required python_requires=">=3.10", # install_requires=["tree-sitter==0.22.3", # "tree-sitter-usfm3==3.0.0-beta.7", diff --git a/py-usfm-parser/src/usfm_grammar/__init__.py b/py-usfm-parser/src/usfm_grammar/__init__.py index 5ea61e2d..e628747e 100755 --- a/py-usfm-parser/src/usfm_grammar/__init__.py +++ b/py-usfm-parser/src/usfm_grammar/__init__.py @@ -9,4 +9,4 @@ Validator = validator.Validator -__version__ = "3.0.0-beta.16" +__version__ = "3.0.0-beta.17" diff --git a/py-usfm-parser/src/usfm_grammar/usfm_parser.py b/py-usfm-parser/src/usfm_grammar/usfm_parser.py index ceca210c..773e5dd2 100755 --- a/py-usfm-parser/src/usfm_grammar/usfm_parser.py +++ b/py-usfm-parser/src/usfm_grammar/usfm_parser.py @@ -233,7 +233,7 @@ def to_usx(self, ignore_errors=False): usx_root = etree.Element("usx") - usx_root.set("version", "3.0") + usx_root.set("version", "3.1") try: usx_generator = USXGenerator(USFM_LANGUAGE, self.usfm_bytes, usx_root) usx_generator.node_2_usx(self.syntax_tree, usx_root) diff --git a/py-usfm-parser/src/usfm_grammar/validator.py b/py-usfm-parser/src/usfm_grammar/validator.py index 90b04bbf..cff4853b 100644 --- a/py-usfm-parser/src/usfm_grammar/validator.py +++ b/py-usfm-parser/src/usfm_grammar/validator.py @@ -158,6 +158,18 @@ def auto_fix_usfm(self, usfm, fixed=False): #pylint: disable=R0912,R0915 # self.modified_usfm = re.sub(to_replace, repalcement, self.modified_usfm) self.modified_usfm = self.modified_usfm.replace(to_replace, repalcement) changed = True + elif error.is_error and \ + (not error_text.startswith("\\")) and \ + error.prev_sibling.type == "chapter" and \ + "paragraph" not in [ch.type for ch in error.children]: + # print("match 7.1") + start = error.prev_sibling.start_byte + end = error.start_byte + to_replace = self.usfm_bytes[start:end].decode('utf-8') + repalcement = to_replace+"\\p\n" + # self.modified_usfm = re.sub(to_replace, repalcement, self.modified_usfm) + self.modified_usfm = self.modified_usfm.replace(to_replace, repalcement) + changed = True # Stray slash not with a valid marker elif error_text.startswith("\\") and \ (not re.match(valid_markers_pattern, error_text)): diff --git a/tests/autofix/b_without_p.usfm b/tests/autofix/b_without_p.usfm new file mode 100644 index 00000000..e63f640a --- /dev/null +++ b/tests/autofix/b_without_p.usfm @@ -0,0 +1,24 @@ +\id GEN genesis Some desc +\c 1 +\p +\v 1 test verse +\s some poem +\q1 +first line +\b +\q1 +\v 2 more lines +\b +remaining verses +\v 3 text follows +\c 2 +\p +\v 1 test verse +\s some poem +\q1 +first line +\b +\q1 +\v 2 more lines +\b +\v 3 verse follows \ No newline at end of file diff --git a/tree-sitter-usfm3/grammar.js b/tree-sitter-usfm3/grammar.js index 65d4dea7..f7c3b3de 100644 --- a/tree-sitter-usfm3/grammar.js +++ b/tree-sitter-usfm3/grammar.js @@ -228,7 +228,11 @@ module.exports = grammar({ r: $ => seq("\\r ", $.text), // ocurs under c too sp: $ => seq("\\sp ", $.text), - d: $ => seq("\\d ", $.text), + d: $ => prec.right(0, seq("\\d ", repeat1(choice($.text, + $.footnote, $.crossref, + $._characterMarker, + $.fig, + )))), sdBlock: $ => prec.right(0, repeat1($.sd)), sd: $ => seq($.sdTag), sdTag: $ => seq("\\sd", optional($.numberedLevelMax4), $._spaceOrLine), diff --git a/tree-sitter-usfm3/package-lock.json b/tree-sitter-usfm3/package-lock.json index 1a2fb3c1..bdf56f42 100644 --- a/tree-sitter-usfm3/package-lock.json +++ b/tree-sitter-usfm3/package-lock.json @@ -1,12 +1,12 @@ { "name": "tree-sitter-usfm3", - "version": "3.0.0-beta.16", + "version": "3.0.0-beta.17", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "tree-sitter-usfm3", - "version": "3.0.0-beta.16", + "version": "3.0.0-beta.17", "hasInstallScript": true, "license": "MIT License", "dependencies": { diff --git a/tree-sitter-usfm3/package.json b/tree-sitter-usfm3/package.json index 28bb0498..b0d3455c 100644 --- a/tree-sitter-usfm3/package.json +++ b/tree-sitter-usfm3/package.json @@ -1,6 +1,6 @@ { "name": "tree-sitter-usfm3", - "version": "3.0.0-beta.16", + "version": "3.0.0-beta.17", "description": "Grammar representation and parser for USFM language using tree-sitter", "main": "bindings/node", "types": "bindings/node", diff --git a/tree-sitter-usfm3/pyproject.toml b/tree-sitter-usfm3/pyproject.toml index 3949193e..55723eac 100644 --- a/tree-sitter-usfm3/pyproject.toml +++ b/tree-sitter-usfm3/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "tree-sitter-usfm3" description = "Usfm3 grammar for tree-sitter" -version = "3.0.0-beta.16" +version = "3.0.0-beta.17" keywords = ["incremental", "parsing", "tree-sitter", "usfm3"] classifiers = [ "Intended Audience :: Developers", diff --git a/web-usfm-parser/README.md b/web-usfm-parser/README.md index 32f9b5f0..1e4479c1 100644 --- a/web-usfm-parser/README.md +++ b/web-usfm-parser/README.md @@ -23,10 +23,10 @@ function App() { useEffect(() => { const initParser = async () => { - await USFMParser.init("https://cdn.jsdelivr.net/npm/usfm-grammar-web@3.0.0-beta.16/tree-sitter-usfm.wasm", - "https://cdn.jsdelivr.net/npm/usfm-grammar-web@3.0.0-beta.16/tree-sitter.wasm"); - await Validator.init("https://cdn.jsdelivr.net/npm/usfm-grammar-web@3.0.0-beta.16/tree-sitter-usfm.wasm", - "https://cdn.jsdelivr.net/npm/usfm-grammar-web@3.0.0-beta.16/tree-sitter.wasm"); + await USFMParser.init("https://cdn.jsdelivr.net/npm/usfm-grammar-web@3.0.0-beta.17/tree-sitter-usfm.wasm", + "https://cdn.jsdelivr.net/npm/usfm-grammar-web@3.0.0-beta.17/tree-sitter.wasm"); + await Validator.init("https://cdn.jsdelivr.net/npm/usfm-grammar-web@3.0.0-beta.17/tree-sitter-usfm.wasm", + "https://cdn.jsdelivr.net/npm/usfm-grammar-web@3.0.0-beta.17/tree-sitter.wasm"); }; initParser(); @@ -60,13 +60,13 @@ It can be used directly in the HTML script tag too. Please ensure its dependenci ```html