* Break-out RFC 7950 Section 6.1 tokenization

  * This enables full string lexical parsing of some rules previously not fully compliant, including:
    * refine
    * user-augment
    * if-feature
  * Also solves previous tokenization issues

CHANGELOG.md

@@ -35,6 +35,15 @@
 * [3.3.2](#332) Aug 27 2017
 * [3.3.1](#331) June 7 2017
 
+## Refine branch
+
+* Break-out RFC 7950 Section 6.1 tokenization
+  * This enables full string lexical parsing of some rules previously not fully compliant, including:
+    * refine, uses-augment, augment, if-feature, type, base.
+  * Also solves previous tokenization issues
+    * [String concatenation in YANG model leads to syntax error ](https://github.com/clicon/clixon/issues/265)
+    * [Can't use + symbol in the enum statement without quotes](https://github.com/clicon/clixon/issues/241)
+
 ## 5.8.0
 Planned: July 2022
 
@@ -122,6 +131,8 @@ Developers may need to change their code
 
 ### Corrected Bugs
 
+* Fixed: [Can't use + symbol in the enum statement without quotes](https://github.com/clicon/clixon/issues/241)
+* Fixed: [String concatenation in YANG model leads to syntax error ](https://github.com/clicon/clixon/issues/265)
 * Fixed: ["autocli:hide-show" extension cause bug in xmldb_put method #343](https://github.com/clicon/clixon/issues/343)
 * Fixed: [Schema Ambiguity Error with openconfig-system re: NTP](https://github.com/clicon/clixon/issues/334)
 * Fixed: [YANG mandatory statements within case nodes do not work](https://github.com/clicon/clixon/issues/344)

lib/clixon/clixon_yang.h

@@ -229,6 +229,7 @@ int       yang_stats(yang_stmt *y, uint64_t *nrp, size_t *szp);
 yang_stmt *yspec_new(void);
 yang_stmt *ys_new(enum rfc_6020 keyw);
 yang_stmt *ys_prune(yang_stmt *yp, int i);
+int        ys_prune_self(yang_stmt *ys);
 int        ys_free1(yang_stmt *ys, int self);
 int        ys_free(yang_stmt *ys);
 int        ys_cp(yang_stmt *nw, yang_stmt *old);

lib/clixon/clixon_yang_parse_lib.h

@@ -59,6 +59,6 @@ int        yang_spec_parse_file(clicon_handle h, char *filename, yang_stmt *yspe
 int        yang_spec_load_dir(clicon_handle h, char *dir, yang_stmt *yspec);
 int        ys_parse_date_arg(char *datearg, uint32_t *dateint);
 cg_var    *ys_parse(yang_stmt *ys, enum cv_type cvtype);
-int        ys_parse_sub(yang_stmt *ys, char *extra);
+int        ys_parse_sub(yang_stmt *ys, const char *filename, char *extra);
 
 #endif  /* _CLIXON_YANG_LIB_H_ */

lib/src/Makefile.in

@@ -77,7 +77,7 @@ SRC     = clixon_sig.c clixon_uid.c clixon_log.c clixon_err.c clixon_event.c \
 	  clixon_xml.c clixon_xml_io.c clixon_xml_sort.c clixon_xml_map.c clixon_xml_vec.c \
 	  clixon_xml_bind.c clixon_json.c clixon_proc.c \
 	  clixon_yang.c clixon_yang_type.c clixon_yang_module.c \
-	  clixon_yang_parse_lib.c clixon_yang_parse_sub.c \
+	  clixon_yang_parse_lib.c clixon_yang_sub_parse.c \
           clixon_yang_cardinality.c clixon_xml_changelog.c clixon_xml_nsctx.c \
 	  clixon_path.c clixon_validate.c \
 	  clixon_hash.c clixon_options.c clixon_data.c clixon_plugin.c \
@@ -95,7 +95,8 @@ YACCOBJS = lex.clixon_xml_parse.o clixon_xml_parse.tab.o \
             lex.clixon_api_path_parse.o clixon_api_path_parse.tab.o \
             lex.clixon_instance_id_parse.o clixon_instance_id_parse.tab.o \
             lex.clixon_text_syntax_parse.o clixon_text_syntax_parse.tab.o \
-            lex.clixon_if_feature_parse.o clixon_if_feature_parse.tab.o 
+            lex.clixon_yang_sub_parse.o clixon_yang_sub_parse.tab.o \
+            lex.clixon_yang_schemanode_parse.o clixon_yang_schemanode_parse.tab.o 
 
 # Generated src 
 GENSRC  = build.c 
@@ -120,22 +121,23 @@ endif
 
 clean:
 	rm -f $(OBJS) $(MYLIBLINK) $(MYLIBSTATIC) $(MYLIBDYNAMIC) $(GENOBJS) $(GENSRC) *.core
-	rm -f clixon_xml_parse.tab.[ch] clixon_xml_parse.yy.[co]
-	rm -f clixon_yang_parse.tab.[ch] clixon_yang_parse.[co]
-	rm -f clixon_json_parse.tab.[ch] clixon_json_parse.[co]
-	rm -f clixon_xpath_parse.tab.[ch] clixon_xpath_parse.[co]
-	rm -f clixon_api_path_parse.tab.[ch] clixon_api_path_parse.[co]
-	rm -f clixon_instance_id_parse.tab.[ch] clixon_instance_id_parse.[co]
-	rm -f clixon_text_syntax_parse.tab.[ch] clixon_text_syntax_parse.[co]
-	rm -f clixon_if_feature_parse.tab.[ch] clixon_if_feature_parse.[co]
+	rm -f clixon_xml_parse.tab.[ch] clixon_xml_parse.[o]
+	rm -f clixon_yang_parse.tab.[ch] clixon_yang_parse.[o]
+	rm -f clixon_json_parse.tab.[ch] clixon_json_parse.[o]
+	rm -f clixon_xpath_parse.tab.[ch] clixon_xpath_parse.[o]
+	rm -f clixon_api_path_parse.tab.[ch] clixon_api_path_parse.[o]
+	rm -f clixon_instance_id_parse.tab.[ch] clixon_instance_id_parse.[o]
+	rm -f clixon_text_syntax_parse.tab.[ch] clixon_text_syntax_parse.[o]
+	rm -f clixon_yang_sub_parse.tab.[ch] clixon_yang_sub_parse.[o]
+	rm -f clixon_yang_schemanode_parse.tab.[ch] clixon_yang_schemanode_parse.[o]
 	rm -f lex.clixon_xml_parse.c
 	rm -f lex.clixon_yang_parse.c
 	rm -f lex.clixon_json_parse.c
 	rm -f lex.clixon_xpath_parse.c
 	rm -f lex.clixon_api_path_parse.c
 	rm -f lex.clixon_instance_id_parse.c
 	rm -f lex.clixon_text_syntax_parse.c
-	rm -f lex.clixon_if_feature_parse.c
+	rm -f lex.clixon_yang_sub_parse.c
 	rm -f *.gcda *.gcno *.gcov # coverage
 
 #############################################################################
@@ -241,17 +243,30 @@ clixon_text_syntax_parse.tab.c:	clixon_text_syntax_parse.tab.h
 lex.clixon_text_syntax_parse.o : lex.clixon_text_syntax_parse.c clixon_text_syntax_parse.tab.h
 	$(CC) $(INCLUDES) $(CPPFLAGS) $(CFLAGS) -Wno-error -c $<
 
-# if-feature-expr parser
-lex.clixon_if_feature_parse.c : clixon_if_feature_parse.l clixon_if_feature_parse.tab.h
-	$(LEX) -Pclixon_if_feature_parse clixon_if_feature_parse.l # -d is debug
+# yang-sub parser
+lex.clixon_yang_sub_parse.c : clixon_yang_sub_parse.l clixon_yang_sub_parse.tab.h
+	$(LEX) -Pclixon_yang_sub_parse clixon_yang_sub_parse.l # -d is debug
 
-clixon_if_feature_parse.tab.h: clixon_if_feature_parse.y
-	$(YACC) -l -d -b clixon_if_feature_parse -p clixon_if_feature_parse clixon_if_feature_parse.y # -t is debug
+clixon_yang_sub_parse.tab.h: clixon_yang_sub_parse.y
+	$(YACC) -l -d -b clixon_yang_sub_parse -p clixon_yang_sub_parse clixon_yang_sub_parse.y # -t is debug
 
 # extra rule to avoid parallell yaccs
-clixon_if_feature_parse.tab.c:	clixon_if_feature_parse.tab.h
+clixon_yang_sub_parse.tab.c:	clixon_yang_sub_parse.tab.h
 
-lex.clixon_if_feature_parse.o : lex.clixon_if_feature_parse.c clixon_if_feature_parse.tab.h
+lex.clixon_yang_sub_parse.o : lex.clixon_yang_sub_parse.c clixon_yang_sub_parse.tab.h
+	$(CC) $(INCLUDES) $(CPPFLAGS) $(CFLAGS) -Wno-error -c $<
+
+# descendant-schema-nodeid parser
+lex.clixon_yang_schemanode_parse.c : clixon_yang_schemanode_parse.l clixon_yang_schemanode_parse.tab.h
+	$(LEX) -Pclixon_yang_schemanode_parse clixon_yang_schemanode_parse.l # -d is debug
+
+clixon_yang_schemanode_parse.tab.h: clixon_yang_schemanode_parse.y
+	$(YACC) -l -d -b clixon_yang_schemanode_parse -p clixon_yang_schemanode_parse clixon_yang_schemanode_parse.y # -t is debug
+
+# extra rule to avoid parallell yaccs
+clixon_yang_schemanode_parse.tab.c:	clixon_yang_schemanode_parse.tab.h
+
+lex.clixon_yang_schemanode_parse.o : lex.clixon_yang_schemanode_parse.c clixon_yang_schemanode_parse.tab.h
 	$(CC) $(INCLUDES) $(CPPFLAGS) $(CFLAGS) -Wno-error -c $<
 
 distclean: clean

lib/src/clixon_text_syntax_parse.l

@@ -94,7 +94,6 @@ int clixon_text_syntax_parsewrap(void)
 <INITIAL>[^\n\r \t\[\]\{\}\;\"]+  {
                             clixon_text_syntax_parselval.string = strdup(yytext);
                             return TOKEN; }
-<INITIAL>.                { return -1; }
 
 <COMMENT>\n               { _TS->ts_linenum++; BEGIN(_TS->ts_lex_state);}
 <COMMENT><<EOF>>          { return MY_EOF; }
@@ -106,7 +105,6 @@ int clixon_text_syntax_parsewrap(void)
 <STRING>\"                { BEGIN(_TS->ts_lex_state); return *yytext; }
 <STRING>[^\n\"]+          { clixon_text_syntax_parselval.string = strdup(yytext);
                             return TOKEN; }
-<STRING>.                 { return -1; }
 
 %%
 

lib/src/clixon_xml_nsctx.c

@@ -95,7 +95,7 @@ xml_nsctx_namespace_netconf_default(clicon_handle h)
 
 /*! Create and initialize XML namespace context
  * @param[in] prefix    Namespace prefix, or NULL for default
- * @param[in] ns	    Set this namespace. If NULL create empty nsctx
+ * @param[in] ns        Set this namespace. If NULL create empty nsctx
  * @retval    nsc       Return namespace context in form of a cvec
  * @retval    NULL      Error
  * @code

lib/src/clixon_xml_parse.l

@@ -139,7 +139,6 @@ ncname     {namestart}{namechar}*
 <STATEA>\r            { clixon_xml_parselval.string = "\n";return WHITESPACE; }
 <STATEA>\n            { clixon_xml_parselval.string = "\n"; _XY->xy_linenum++;return WHITESPACE; }
 <STATEA>[^&\r\n \t\<]+ { clixon_xml_parselval.string = yytext; return CHARDATA; /* Optimized */}
-<STATEA>.             { clixon_xml_parselval.string = yytext; return CHARDATA; }
 
 	/* @see xml_chardata_encode */
 <AMPERSAND>"amp;"    { BEGIN(_XY->xy_lex_state); clixon_xml_parselval.string = "&"; return CHARDATA;}

lib/src/clixon_yang.c

@@ -80,7 +80,7 @@
 #include "clixon_data.h"
 #include "clixon_options.h"
 #include "clixon_yang_parse.h"
-#include "clixon_yang_parse_sub.h"
+#include "clixon_yang_sub_parse.h"
 #include "clixon_yang_parse_lib.h"
 #include "clixon_yang_cardinality.h"
 #include "clixon_yang_type.h"
@@ -673,6 +673,7 @@ ys_free1(yang_stmt *ys,
  * @retval     NULL No such node, nothing done
  * @retval     yc   returned orphaned yang node
  * @see ys_free Deallocate yang node
+ * @see ys_prune_self Prune child itself
  * @note Do not call this in a loop of yang children (unless you know what you are doing)
  */
 yang_stmt *
@@ -705,7 +706,7 @@ ys_prune(yang_stmt *yp,
  * @see ys_free Deallocate yang node
  * @note Do not call this in a loop of yang children (unless you know what you are doing)
  */
-static int
+int
 ys_prune_self(yang_stmt *ys)
 {
     int        retval = -1;
@@ -2800,15 +2801,19 @@ yang_features(clicon_handle h,
     int        i;
     int        j;
     yang_stmt *ys = NULL;
+    yang_stmt *ymod;
+    const char *mainfile = NULL;
     int        ret;
 
     i = 0;
     while (i<yt->ys_len){
 	ys = yt->ys_stmt[i];
 	if (ys->ys_keyword == Y_IF_FEATURE){
 	    /* Parse the if-feature-expr string using yang sub-parser */
+	    if ((ymod = ys_module(ys)) != NULL)
+		mainfile = yang_filename_get(ymod);
 	    ret = 0;
-	    if (yang_if_feature_parse(yang_argument_get(ys), ys, 1, &ret) < 0)
+	    if (yang_subparse(yang_argument_get(ys), ys, YA_IF_FEATURE, mainfile, 1, &ret) < 0)
 		goto done;
 	    clicon_debug(1, "%s %s %d", __FUNCTION__, yang_argument_get(ys), ret);
 	    if (ret == 0)
@@ -2938,6 +2943,7 @@ yang_datanode(yang_stmt *ys)
 /*! All the work for schema_nodeid functions both absolute and descendant
  *
  * @param[in]  yn    Yang node. For absolute schemanodeids this should be a module, otherwise any yang
+ * @param[in]  yspec Yang spec (Cornsercase if yn is pruned)
  * @param[in]  cvv   Schema-node path encoded as a name/value pair list.
  * @param[in]  nsc   Namespace context from yang for the prefixes (names) of cvv
  * @param[out] yres  Result yang statement node, or NULL if not found
@@ -2954,6 +2960,7 @@ yang_datanode(yang_stmt *ys)
  */
 static int
 schema_nodeid_iterate(yang_stmt    *yn,
+		      yang_stmt    *yspec,
 		      cvec         *nodeid_cvv,
 		      cvec         *nsc,
 		      yang_stmt   **yres)
@@ -2966,9 +2973,7 @@ schema_nodeid_iterate(yang_stmt    *yn,
     yang_stmt       *yp;
     cg_var          *cv;
     char            *ns;
-    yang_stmt       *yspec;
 
-    yspec = ys_spec(yn);
     yp = yn;
     /* Iterate over node identifiers /prefix:id/... */
     cv = NULL;    
@@ -3050,8 +3055,11 @@ yang_abs_schema_nodeid(yang_stmt    *yn,
     yang_stmt    *ymod;
     char         *str;
 
+    if ((yspec = ys_spec(yn)) == NULL){
+	clicon_err(OE_YANG, EINVAL, "No yang spec");
+	goto done;
+    }
     *yres = NULL;
-    yspec = ys_spec(yn);
     /* check absolute schema_nodeid */
     if (schema_nodeid[0] != '/'){
 	clicon_err(OE_YANG, EINVAL, "absolute schema nodeid should start with /");
@@ -3099,7 +3107,7 @@ yang_abs_schema_nodeid(yang_stmt    *yn,
 	goto done;
     }
     /* Iterate through cvv to find schemanode using ymod as starting point (since it is absolute) */
-    if (schema_nodeid_iterate(ymod, nodeid_cvv, nsc, yres) < 0)
+    if (schema_nodeid_iterate(ymod, yspec, nodeid_cvv, nsc, yres) < 0)
 	goto done;
  ok:
     retval = 0;
@@ -3112,9 +3120,10 @@ yang_abs_schema_nodeid(yang_stmt    *yn,
 }
 
 /*! Given a descendant schema-nodeid (eg a/b/c) find matching yang spec  
- * @param[in]  yn            Yang node
+ * @param[in]  yn            Yang node (must be part of yspec tree, cannot be "dangling")
  * @param[in]  schema_nodeid Descendant schema-node-id, ie a/b
  * @param[in]  keyword       A schemode of this type, or -1 if any
+ * @param[in]  ns            Namespace context
  * @param[out] yres          First yang node matching schema nodeid
  * @retval     0             OK
  * @retval    -1             Error, with clicon_err called
@@ -3130,12 +3139,17 @@ yang_desc_schema_nodeid(yang_stmt    *yn,
     cvec         *nodeid_cvv = NULL;
     cg_var       *cv;
     char         *str;
+    yang_stmt    *yspec;
     cvec         *nsc = NULL;
 
     if (schema_nodeid == NULL || strlen(schema_nodeid) == 0){
 	clicon_err(OE_YANG, EINVAL, "nodeid is empty");
 	goto done;
     }
+    if ((yspec = ys_spec(yn)) == NULL){
+	clicon_err(OE_YANG, EINVAL, "No yang spec");
+	goto done;
+    }
     *yres = NULL;
     /* check absolute schema_nodeid */
     if (schema_nodeid[0] == '/'){
@@ -3161,11 +3175,13 @@ yang_desc_schema_nodeid(yang_stmt    *yn,
 	    cv_name_set(cv, NULL);
 	}
     }
-    /* Make a namespace context from yang for the prefixes (names) of nodeid_cvv */
+    /* Make a namespace context from yang for the prefixes (names) of nodeid_cvv 
+     * Requires yn exist in hierarchy
+     */
     if (xml_nsctx_yang(yn, &nsc) < 0)
 	goto done;
     /* Iterate through cvv to find schemanode using yn as relative starting point */
-    if (schema_nodeid_iterate(yn, nodeid_cvv, nsc, yres) < 0)
+    if (schema_nodeid_iterate(yn, yspec, nodeid_cvv, nsc, yres) < 0)
 	goto done;
  ok:
     retval = 0;

lib/src/clixon_yang_parse.l

@@ -119,7 +119,6 @@ identifier      [A-Za-z_][A-Za-z0-9_\-\.]*
 <KEYWORD>anydata          { BEGIN(STRARG);  return K_ANYDATA; }
 <KEYWORD>anyxml           { BEGIN(STRARG);  return K_ANYXML; }
 <KEYWORD>argument         { BEGIN(STRARG);  return K_ARGUMENT; }
-<KEYWORD>base             { BEGIN(STRARG);  return K_BASE; }
 <KEYWORD>belongs-to       { BEGIN(STRARG);  return K_BELONGS_TO; }
 <KEYWORD>bit              { BEGIN(STRARG);  return K_BIT; }
 <KEYWORD>case             { BEGIN(STRARG);  return K_CASE; }
@@ -137,12 +136,9 @@ identifier      [A-Za-z_][A-Za-z0-9_\-\.]*
 <KEYWORD>module           { BEGIN(STRARG);  return K_MODULE; }
 <KEYWORD>notification     { BEGIN(STRARG);  return K_NOTIFICATION; }
 <KEYWORD>prefix           { BEGIN(STRARG);  return K_PREFIX; }
-<KEYWORD>refine           { BEGIN(STRARG);  return K_REFINE; }
 <KEYWORD>rpc              { BEGIN(STRARG);  return K_RPC; }
 <KEYWORD>submodule        { BEGIN(STRARG);  return K_SUBMODULE; }
-<KEYWORD>type             { BEGIN(STRARG);  return K_TYPE; }
 <KEYWORD>typedef          { BEGIN(STRARG);  return K_TYPEDEF; }
-<KEYWORD>uses             { BEGIN(STRARG);  return K_USES; }
 
  /* RFC 7950 keywords using boolean string arguments */
 <KEYWORD>config           { BEGIN(BOOLEAN); return K_CONFIG; }
@@ -157,6 +153,7 @@ identifier      [A-Za-z_][A-Za-z0-9_\-\.]*
 
  /* RFC 7950 keywords using strings  */
 <KEYWORD>augment          { BEGIN(STRING);  return K_AUGMENT; }
+<KEYWORD>base             { BEGIN(STRING);  return K_BASE; }
 <KEYWORD>contact          { BEGIN(STRING);  return K_CONTACT; }
 <KEYWORD>default          { BEGIN(STRING);  return K_DEFAULT; }
 <KEYWORD>description      { BEGIN(STRING);  return K_DESCRIPTION; }
@@ -178,13 +175,16 @@ identifier      [A-Za-z_][A-Za-z0-9_\-\.]*
 <KEYWORD>path             { BEGIN(STRING);  return K_PATH; }
 <KEYWORD>pattern          { BEGIN(STRING);  return K_PATTERN; }
 <KEYWORD>presence         { BEGIN(STRING);  return K_PRESENCE; }
+<KEYWORD>type             { BEGIN(STRING);  return K_TYPE; }
 <KEYWORD>unique           { BEGIN(STRING);  return K_UNIQUE; }
 <KEYWORD>range            { BEGIN(STRING);  return K_RANGE; }
 <KEYWORD>reference        { BEGIN(STRING);  return K_REFERENCE; }
+<KEYWORD>refine           { BEGIN(STRING);  return K_REFINE; }
 <KEYWORD>revision         { BEGIN(STRING);  return K_REVISION; }
 <KEYWORD>revision-date    { BEGIN(STRING);  return K_REVISION_DATE; }
 <KEYWORD>status           { BEGIN(STRING);  return K_STATUS; }
 <KEYWORD>units            { BEGIN(STRING);  return K_UNITS; }
+<KEYWORD>uses             { BEGIN(STRING);  return K_USES; }
 <KEYWORD>when             { BEGIN(STRING);  return K_WHEN; }
 <KEYWORD>yang-version     { BEGIN(STRING);  return K_YANG_VERSION; }
 <KEYWORD>:                { return *yytext; }
@@ -239,15 +239,14 @@ identifier      [A-Za-z_][A-Za-z0-9_\-\.]*
 <STRARG>;                  { BEGIN(KEYWORD); return *yytext; }
 <STRARG>{identifier}       { clixon_yang_parselval.string = strdup(yytext);
                              return IDENTIFIER;}
-
 <STRARG>.                  { return *yytext; }
 
 <STRING>\{                { BEGIN(KEYWORD); return *yytext; }
 <STRING>;                 { BEGIN(KEYWORD); return *yytext; }
 <STRING>\"                { _YY->yy_lex_string_state =STRING; BEGIN(STRINGDQ); return *yytext; }
 <STRING>\'                { _YY->yy_lex_string_state =STRING; BEGIN(STRINGSQ); return *yytext; }
 <STRING>\+                { return *yytext; }
-<STRING>[^\"\'\+\{\;\n \t\r]+    { clixon_yang_parselval.string = strdup(yytext); /* XXX [.]+ */
+<STRING>[^\"\'\{\;\n \t\r]+ { clixon_yang_parselval.string = strdup(yytext); /* XXX [.]+ */
                             return CHARS;}
 
 <STRINGDQ>\\              { _YY->yy_lex_state = STRINGDQ; BEGIN(DQESC); }