Support nanosecond resolution for pkgdb mtime.

When running the test suite, often the pkgdb will be updated and pkgin
executed within the same second.  This would result in pkgin's database
being out of date.  The support for the various ways that nanosecond
resolution is implemented across OS has been taken from libarchive.

This commit also starts to introduce prepared statements and more direct
database access for queries rather than going via the pkgindb_doquery
wrapper.  This results in faster database access and avoids
unnecessary string copies and conversions.

Adjust the SQL queries to be explicit about storing entries in the pkgdb
table, and optimise to only retrieve the most recent.

config.h.in

@@ -135,6 +135,21 @@
 /* Define to 1 if you have the `strtol' function. */
 #undef HAVE_STRTOL
 
+/* Define to 1 if `st_mtimespec.tv_nsec' is a member of `struct stat'. */
+#undef HAVE_STRUCT_STAT_ST_MTIMESPEC_TV_NSEC
+
+/* Define to 1 if `st_mtime_n' is a member of `struct stat'. */
+#undef HAVE_STRUCT_STAT_ST_MTIME_N
+
+/* Define to 1 if `st_mtime_usec' is a member of `struct stat'. */
+#undef HAVE_STRUCT_STAT_ST_MTIME_USEC
+
+/* Define to 1 if `st_mtim.tv_nsec' is a member of `struct stat'. */
+#undef HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC
+
+/* Define to 1 if `st_umtime' is a member of `struct stat'. */
+#undef HAVE_STRUCT_STAT_ST_UMTIME
+
 /* Define to 1 if you have the <sys/cdefs.h> header file. */
 #undef HAVE_SYS_CDEFS_H
 

configure

@@ -1947,6 +1947,63 @@ $as_echo "$ac_res" >&6; }
   eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
 
 } # ac_fn_c_find_uintX_t
+
+# ac_fn_c_check_member LINENO AGGR MEMBER VAR INCLUDES
+# ----------------------------------------------------
+# Tries to find if the field MEMBER exists in type AGGR, after including
+# INCLUDES, setting cache variable VAR accordingly.
+ac_fn_c_check_member ()
+{
+  as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
+  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2.$3" >&5
+$as_echo_n "checking for $2.$3... " >&6; }
+if eval \${$4+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+$5
+int
+main ()
+{
+static $2 ac_aggr;
+if (ac_aggr.$3)
+return 0;
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+  eval "$4=yes"
+else
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+$5
+int
+main ()
+{
+static $2 ac_aggr;
+if (sizeof ac_aggr.$3)
+return 0;
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+  eval "$4=yes"
+else
+  eval "$4=no"
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+eval ac_res=\$$4
+	       { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5
+$as_echo "$ac_res" >&6; }
+  eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
+
+} # ac_fn_c_check_member
 cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
@@ -4762,6 +4819,58 @@ _ACEOF
   esac
 
 
+# Check for high-resolution timestamps in struct stat (from libarchive).
+ac_fn_c_check_member "$LINENO" "struct stat" "st_mtimespec.tv_nsec" "ac_cv_member_struct_stat_st_mtimespec_tv_nsec" "$ac_includes_default"
+if test "x$ac_cv_member_struct_stat_st_mtimespec_tv_nsec" = xyes; then :
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_STRUCT_STAT_ST_MTIMESPEC_TV_NSEC 1
+_ACEOF
+
+
+fi
+
+ac_fn_c_check_member "$LINENO" "struct stat" "st_mtim.tv_nsec" "ac_cv_member_struct_stat_st_mtim_tv_nsec" "$ac_includes_default"
+if test "x$ac_cv_member_struct_stat_st_mtim_tv_nsec" = xyes; then :
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC 1
+_ACEOF
+
+
+fi
+
+ac_fn_c_check_member "$LINENO" "struct stat" "st_mtime_n" "ac_cv_member_struct_stat_st_mtime_n" "$ac_includes_default"
+if test "x$ac_cv_member_struct_stat_st_mtime_n" = xyes; then :
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_STRUCT_STAT_ST_MTIME_N 1
+_ACEOF
+
+
+fi
+ # AIX
+ac_fn_c_check_member "$LINENO" "struct stat" "st_umtime" "ac_cv_member_struct_stat_st_umtime" "$ac_includes_default"
+if test "x$ac_cv_member_struct_stat_st_umtime" = xyes; then :
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_STRUCT_STAT_ST_UMTIME 1
+_ACEOF
+
+
+fi
+ # Tru64
+ac_fn_c_check_member "$LINENO" "struct stat" "st_mtime_usec" "ac_cv_member_struct_stat_st_mtime_usec" "$ac_includes_default"
+if test "x$ac_cv_member_struct_stat_st_mtime_usec" = xyes; then :
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_STRUCT_STAT_ST_MTIME_USEC 1
+_ACEOF
+
+
+fi
+ # Hurd
+
 # Extract the first word of "nbsed", so it can be a program name with args.
 set dummy nbsed; ac_word=$2
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5

configure.ac

@@ -73,6 +73,13 @@ AC_TYPE_UINT16_T
 AC_TYPE_UINT64_T
 AC_TYPE_UINT8_T
 
+# Check for high-resolution timestamps in struct stat (from libarchive).
+AC_CHECK_MEMBERS([struct stat.st_mtimespec.tv_nsec])
+AC_CHECK_MEMBERS([struct stat.st_mtim.tv_nsec])
+AC_CHECK_MEMBERS([struct stat.st_mtime_n]) # AIX
+AC_CHECK_MEMBERS([struct stat.st_umtime]) # Tru64
+AC_CHECK_MEMBERS([struct stat.st_mtime_usec]) # Hurd
+
 AC_CHECK_PROG(SED,nbsed,nbsed,sed)
 
 dnl fix rpath for SunOS, is there a cleaner way to do this ?

pkgin.h

@@ -112,6 +112,21 @@
 
 #define TRACE(fmt...) if (tracefp != NULL) fprintf(tracefp, fmt)
 
+/* Support various ways to get nanosecond resolution, or default to 0 */
+#if HAVE_STRUCT_STAT_ST_MTIMESPEC_TV_NSEC
+#define pkgin_nanotime	st_mtimespec.tv_nsec
+#elif HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC
+#define pkgin_nanotime	st_mtim.tv_nsec
+#elif HAVE_STRUCT_STAT_ST_MTIME_N
+#define pkgin_nanotime	st_mtime_n
+#elif HAVE_STRUCT_STAT_ST_UMTIME
+#define pkgin_nanotime	st_umtime
+#elif HAVE_STRUCT_STAT_ST_MTIME_USEC
+#define pkgin_nanotime	st_mtime.usec
+#else
+#define pkgin_nanotime	0
+#endif
+
 /**
  * \struct Sumfile
  * \brief Remote pkg_summary information

pkgin.sql

@@ -1,5 +1,6 @@
 CREATE TABLE [PKGDB] (
-	"PKGDB_MTIME" INTEGER
+	"PKGDB_MTIME" INTEGER,
+	"PKGDB_NTIME" INTEGER
 );
 
 CREATE TABLE [REPOS] (

pkgindb.c

@@ -246,7 +246,8 @@ pkgindb_open(void)
 			errx(EXIT_FAILURE, "cannot create database: %s",
 			    sqlite3_errmsg(pdb));
 	} else {
-		if (pkgindb_doquery(COMPAT_CHECK, NULL, NULL) != PDB_OK) {
+		if (pkgindb_doquery("SELECT PKGDB_NTIME FROM PKGDB;",
+		    NULL, NULL) != PDB_OK) {
 			if (unlink(pkgin_sqldb) < 0)
 				err(EXIT_FAILURE, "cannot recreate database");
 			goto recreate;
@@ -268,32 +269,74 @@ pkgindb_close()
 	sqlite3_close(pdb);
 }
 
+static void
+pkgindb_sqlfail(void)
+{
+	pkgindb_close();
+
+	errx(EXIT_FAILURE, "SQL query failed, see %s", pkgin_sqllog);
+}
+
 int
-pkg_db_mtime()
+pkg_db_mtime(void)
 {
+	sqlite3_stmt	*stmt;
 	struct stat	st;
-	time_t	   	db_mtime = 0;
-	char		str_mtime[20], buf[BUFSIZ];
+	time_t	   	db_mtime;
+	long		db_ntime;
+	int		rc;
 
-	/* no pkgdb file */
+	/* No pkgdb, just return up-to-date so we can start installing. */
 	if (stat(pkgdb_get_dir(), &st) < 0)
 		return 0;
 
-	str_mtime[0] = '\0';
+	/*
+	 * Fetch only the most recent rowid.
+	 */
+	curquery = "SELECT PKGDB_MTIME, PKGDB_NTIME FROM PKGDB "
+		   "ORDER BY ROWID DESC LIMIT 1;";
 
-	pkgindb_doquery("SELECT PKGDB_MTIME FROM PKGDB;",
-		pdb_get_value, str_mtime);
+	if (sqlite3_prepare_v2(pdb, curquery, -1, &stmt, NULL) != SQLITE_OK)
+		pkgindb_sqlfail();
 
-	if (str_mtime[0] != '\0')
-		db_mtime = (time_t)strtol(str_mtime, (char **)NULL, 10);
+	rc = sqlite3_step(stmt);
 
-	/* mtime is up to date */
-	if (db_mtime == st.st_mtime)
+	if (rc == SQLITE_ROW) {
+		db_mtime = sqlite3_column_int64(stmt, 0);
+		db_ntime = sqlite3_column_int64(stmt, 1);
+	} else if (rc == SQLITE_DONE) {
+		db_mtime = 0;
+		db_ntime = 0;
+	} else {
+		/* This shouldn't happen, abort and investigate */
+		pkgindb_sqlfail();
+	}
+
+	sqlite3_finalize(stmt);
+
+	/* Databases matched pkgdb, we're up-to-date */
+	if (db_mtime == st.st_mtime && db_ntime == st.pkgin_nanotime)
 		return 0;
 
-	snprintf(buf, BUFSIZ, UPDATE_PKGDB_MTIME, (long long)st.st_mtime);
-	/* update mtime */
-	pkgindb_doquery(buf, NULL, NULL);
+	/*
+	 * Update database to current mtime and request a refresh.
+	 */
+	curquery = "INSERT INTO PKGDB (PKGDB_MTIME, PKGDB_NTIME) "
+		   "VALUES (?, ?);";
+
+	if (sqlite3_prepare_v2(pdb, curquery, -1, &stmt, NULL) != SQLITE_OK)
+		pkgindb_sqlfail();
+
+	if (sqlite3_bind_int64(stmt, 1, st.st_mtime) != SQLITE_OK)
+		pkgindb_sqlfail();
+
+	if (sqlite3_bind_int64(stmt, 2, st.pkgin_nanotime) != SQLITE_OK)
+		pkgindb_sqlfail();
+
+	if (sqlite3_step(stmt) != SQLITE_DONE)
+		pkgindb_sqlfail();
+
+	sqlite3_finalize(stmt);
 
 	return 1;
 }

pkgindb.h

@@ -59,7 +59,6 @@ extern const char NOKEEP_LOCAL_PKGS[];
 extern const char KEEP_LOCAL_PKGS[];
 extern const char PKG_URL[];
 extern const char DELETE_EMPTY_ROWS[];
-extern const char UPDATE_PKGDB_MTIME[];
 extern const char EXISTS_REPO[];
 extern const char INSERT_REPO[];
 extern const char SELECT_REPO_URLS[];
@@ -72,7 +71,6 @@ extern const char UNIQUE_PKG[];
 extern const char EXPORT_KEEP_LIST[];
 extern const char GET_PKGNAME_BY_PKGPATH[];
 extern const char GET_ORPHAN_PACKAGES[];
-extern const char COMPAT_CHECK[];
 extern const char SHOW_ALL_CATEGORIES[];
 
 #define LOCAL_PKG "LOCAL_PKG"

pkgindb_queries.c

@@ -167,9 +167,6 @@ const char PKG_URL[] =
 const char DELETE_EMPTY_ROWS[] =
 	"DELETE FROM REMOTE_PKG WHERE PKGNAME IS NULL;";
 
-const char UPDATE_PKGDB_MTIME[] =
-	"REPLACE INTO PKGDB (PKGDB_MTIME) VALUES (%lld);";
-
 const char SELECT_REPO_URLS[] =
 	"SELECT REPO_URL FROM REPOS;";
 
@@ -209,9 +206,6 @@ const char GET_ORPHAN_PACKAGES[] =
 	"SELECT FULLPKGNAME FROM LOCAL_PKG WHERE PKG_KEEP IS NULL AND "
 	"PKGNAME NOT IN (SELECT LOCAL_DEPS_PKGNAME FROM LOCAL_DEPS);";
 
-const char COMPAT_CHECK[] =
-	"SELECT FULLPKGNAME FROM REMOTE_PKG LIMIT 1;";
-
 const char SHOW_ALL_CATEGORIES[] =
 	"SELECT DISTINCT CATEGORIES FROM REMOTE_PKG WHERE "
 	"CATEGORIES NOT LIKE '%% %%' ORDER BY CATEGORIES DESC;";