Skip to content

Commit

Permalink
Ensure single quotes are used for string literals.
Browse files Browse the repository at this point in the history
SQLite 3.27.0 introduced a change in behaviour and started issuing
warnings to the error log when double quotes were being used for string
literals.  This happens to have been the way pkgin would insert every
package into the database, causing huge disk I/O to the sql.log file
when running "pkgin update".

At best this was significantly impacting runtime by orders of magnitude
and wasting disk space with large sql.log files, and at worst this was
reproducibly causing updates to fail entirely with "Short read of
pkg_summary: Lzma library error:  No progress is possible".

The exact details of why the extra logging was causing lzma failures is
still unclear and needs to be looked into to ensure we aren't missing
something else, but this change is demonstrated to avoid the issue.
  • Loading branch information
Jonathan Perkin committed May 20, 2019
1 parent 8ee55a0 commit 91742d8
Show file tree
Hide file tree
Showing 3 changed files with 31 additions and 19 deletions.
2 changes: 1 addition & 1 deletion pkgindb.c
Original file line number Diff line number Diff line change
Expand Up @@ -375,7 +375,7 @@ pkg_sum_mtime(char *repo)
str_mtime[0] = '\0';

snprintf(query, BUFSIZ,
"SELECT REPO_MTIME FROM REPOS WHERE REPO_URL GLOB \'%s*\';",
"SELECT REPO_MTIME FROM REPOS WHERE REPO_URL GLOB '%s*';",
repo);
pkgindb_doquery(query, pdb_get_value, str_mtime);

Expand Down
18 changes: 9 additions & 9 deletions pkgindb_queries.c
Original file line number Diff line number Diff line change
Expand Up @@ -114,9 +114,9 @@ const char LOCAL_PROVIDES[] =
"SELECT LOCAL_PROVIDES_PKGNAME FROM LOCAL_PROVIDES;";

const char KEEP_PKG[] =
"UPDATE LOCAL_PKG SET PKG_KEEP = 1 WHERE PKGNAME = \'%s\';";
"UPDATE LOCAL_PKG SET PKG_KEEP = 1 WHERE PKGNAME = '%s';";
const char UNKEEP_PKG[] =
"UPDATE LOCAL_PKG SET PKG_KEEP = NULL WHERE PKGNAME = \'%s\';";
"UPDATE LOCAL_PKG SET PKG_KEEP = NULL WHERE PKGNAME = '%s';";

const char INSERT_REMOTE_PKG[] =
"INSERT INTO REMOTE_PKG (PKG_ID,FULLPKGNAME,PKGNAME,PKGVERS,BUILD_DATE,"
Expand Down Expand Up @@ -162,7 +162,7 @@ const char KEEP_LOCAL_PKGS[] =
"FROM LOCAL_PKG WHERE PKG_KEEP IS NOT NULL;";

const char PKG_URL[] =
"SELECT REPOSITORY FROM REMOTE_PKG WHERE FULLPKGNAME = \'%s\';";
"SELECT REPOSITORY FROM REMOTE_PKG WHERE FULLPKGNAME = '%s';";

const char DELETE_EMPTY_ROWS[] =
"DELETE FROM REMOTE_PKG WHERE PKGNAME IS NULL;";
Expand All @@ -171,22 +171,22 @@ const char SELECT_REPO_URLS[] =
"SELECT REPO_URL FROM REPOS;";

const char EXISTS_REPO[] =
"SELECT COUNT(*) FROM REPOS WHERE REPO_URL = \'%s\';";
"SELECT COUNT(*) FROM REPOS WHERE REPO_URL = '%s';";

const char INSERT_REPO[] =
"INSERT INTO REPOS (REPO_URL, REPO_MTIME) VALUES (\'%s\', 0);";
"INSERT INTO REPOS (REPO_URL, REPO_MTIME) VALUES ('%s', 0);";

const char UPDATE_REPO_MTIME[] =
"UPDATE REPOS SET REPO_MTIME = %lld WHERE REPO_URL = \'%s\';";
"UPDATE REPOS SET REPO_MTIME = %lld WHERE REPO_URL = '%s';";

const char DELETE_REPO_URL[] =
"DELETE FROM REPOS WHERE REPO_URL = \'%s\';";
"DELETE FROM REPOS WHERE REPO_URL = '%s';";

const char INSERT_SINGLE_VALUE[] =
"INSERT INTO %s (PKG_ID, %s_PKGNAME) VALUES (%d,\"%s\");";
"INSERT INTO %s (PKG_ID, %s_PKGNAME) VALUES (%d,'%s');";

const char INSERT_DEPENDS_VALUES[] =
"INSERT INTO %s (PKG_ID, %s_PKGNAME, %s_DEWEY) VALUES (%d,\"%s\",\"%s\");";
"INSERT INTO %s (PKG_ID, %s_PKGNAME, %s_DEWEY) VALUES (%d,'%s','%s');";

const char UNIQUE_PKG[] =
"SELECT FULLPKGNAME, PKGVERS FROM %s WHERE PKGNAME = '%s';";
Expand Down
30 changes: 21 additions & 9 deletions summary.c
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ prepare_insert(int pkgid, struct Summary sum)

/* insert fields */
SLIST_FOREACH(pi, &inserthead, next) {
snprintf(tmpbuf, sizeof(tmpbuf), ",\"%s\"", pi->field);
snprintf(tmpbuf, sizeof(tmpbuf), ",%s", pi->field);
if (strlcat(querybuf, tmpbuf, sizeof(querybuf)) >= sizeof(querybuf))
goto err;
}
Expand All @@ -221,7 +221,7 @@ prepare_insert(int pkgid, struct Summary sum)

/* insert values */
SLIST_FOREACH(pi, &inserthead, next) {
snprintf(tmpbuf, sizeof(tmpbuf), ",\"%s\"", pi->value);
snprintf(tmpbuf, sizeof(tmpbuf), ",'%s'", pi->value);
if (strlcat(querybuf, tmpbuf, sizeof(querybuf)) >= sizeof(querybuf))
goto err;
}
Expand Down Expand Up @@ -261,7 +261,7 @@ parse_entry(struct Summary sum, int pkgid, char *line)
{
static uint8_t check_machine_arch = 1;
int i;
char *val, *v, *pkg, buf[BUFSIZ];
char *val, *v, *p, *pkg, buf[BUFSIZ], *tmp;

if ((val = strchr(line, '=')) == NULL)
errx(EXIT_FAILURE, "Invalid pkg_info entry: %s", line);
Expand Down Expand Up @@ -336,14 +336,25 @@ parse_entry(struct Summary sum, int pkgid, char *line)
snprintf(buf, BUFSIZ, "%s=", cols.name[i]);

if (strncmp(buf, line, strlen(buf)) == 0) {
tmp = NULL;
/*
* Avoid double quotes in our query by using
* the MySQL-compatible "`" instead.
* Convert single quotes to SQLite compatible ''.
*/
if (strchr(val, '"') != NULL)
for (v = val; *v != '\0'; v++)
if (*v == '"')
*v = '`';
if (strchr(val, '\'') != NULL) {
for (v = p = val; *v != '\0'; v++) {
if (*v != '\'')
continue;

*v = '\0';
if (tmp)
tmp = xasprintf("%s%s''", tmp, p);
else
tmp = xasprintf("%s''", p);
p = v + 1;
}
tmp = xasprintf("%s%s", tmp, p);
val = tmp;
}

/* Split PKGNAME into parts */
if (strncmp(cols.name[i], "PKGNAME", 7) == 0) {
Expand All @@ -364,6 +375,7 @@ parse_entry(struct Summary sum, int pkgid, char *line)
} else
add_to_slist(cols.name[i], val);

free(tmp);
break;
}
}
Expand Down

0 comments on commit 91742d8

Please sign in to comment.